Terminal comands to download and clean Google ngram datasets with a set of regular expressions.

ngram-cleaner.sh

#!/bin/bash

cat lista.txt | while read -r file; do wget "$file" && gzip -dc "${file##*/}" | \
tr '[:upper:]' '[:lower:]' | sed \
-e 's/\(_\?noun_\?\)\|\(_\?adp_\?\)\|\(_\?adv_\?\)\|\(_\?det_\?\)\|\(_\?verb_\?\)\|\(_\?adj_\?\)\|\(_\?end_\?\)\|\(_\?conj_\?\)\|\(S\?_\?pron_\?\)\|\(_\?num_\?\)/ /g' \
-e "s/\([^a-zàéèìòù '.]\)/ /g" \
-e "s/\s\{2,\}/ /g" \
-e "s/\.\{2,\}/./g" \
-e "s/[^a-zàéèìòù.]\{1,\}\.\{1,\}//g" \
-e "s/\s\{2,\}/ /g" \
-e "s/'/' /g" \
-e "s/\s'/' /g" \
-e "s/\(\s\{0,\}[^a-zàéèìòù']\.$\)//g" \
-e "s/\(^\.\{1,\}[^a-zàéèìòù']\{0,\}\s\{0,\}\)//g" \
-e "s/\s\{2,\}/ /g" \
-e 's/\(^\s\{1,\}\)\|\(\s\{1,\}$\)//g' \
| sort | uniq > "${file##*/}_cleaned.txt" && rm "${file##*/}"; done