peel-nlp - series of scripts to turn https://sites.google.com/a/ualberta.ca/opendata/data into https://www.google.com/maps/d/u/0/edit?mid=z92YH-I0oPAM.k4BoSsNm1qP0

frequency_sort.sh

sort corpus_locations.csv | uniq -c | sort -nr > corpus_locations_count.tsv

just_fulltext.sh

apt-get install xpath

for file in $(find . -iname '*.xml')
do
  echo $file
  xpath -e '//note/text()' $file > $file.txt
done

just_locations.sh

for file in $(find . -iname '*.ner')
do
  echo $file
  awk -F 'START:location>|<END' '{print $2}' $file >> corpus_locations.csv
done

ner-markup.sh

for file in $(find . -iname '*.txt')
do
  echo $file
  /root/apache-opennlp-1.5.3/bin/opennlp TokenNameFinder /root/apache-opennlp-1.5.3/bin/en-ner-location.bin <  $file > $file.ner
done