Convert text dataset to wordlist for Turkish language.

dataset_to_wordlist_tr.sh

#!/bin/bash
# -*- coding: utf8 -*-

# USAGE: bash dataset_to_wordlist_tr.sh ./input.txt ./output.txt
# This script takes file clean punctuation, digits and convert capital
# latters to smaller and clean non Turkish characters.
# Finally make sorted uniq wordlist for Turkish language.
# If you want to add or remove character you can change line 20.

# Define Variables for input and output
dataset=$1
output=$2

# Check inputs
if [[ -z "$dataset" || -z "$output" ]]
  then
    echo "No argument supplied. Please give input and output filename."
else
	cat "$dataset" | sed 's/[[:punct:]]/ /g' | sed 's/[0-9]//g' | \
	tr A-Z a-z | sed 's/[^abcdefghijklmnopqrstuvwxyzşıüğçöâîû ]//g' | \
	sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | \
	sed 's/ //g' | sort | uniq > "$output"
fi