text prep for k-nn

text.py

import sys, os
# sys.path.append('./Common')  # noqa
sys.path.append(os.path.abspath('./Common'))

import Common.common as common
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib
from collections import Counter, OrderedDict
import re
matplotlib.style.use('ggplot')

data_file_name = './Chapter01/text_classification/pg10900.txt'
stop_words_file = './Chapter01/text_classification/stopwords.txt'

words = re.findall(r'\w+', open(data_file_name).read().lower())
word_count = Counter(words).most_common(10000)

# (Actual-Min)/(Max-Min)
ordered_word_count = OrderedDict(word_count)
first = list(ordered_word_count.items())[0]
last = list(ordered_word_count.items())[-1]


word_count_scaled = ()
scale = lambda actual: (actual-last[1])/(first[1]-last[1])

for x in word_count:
    try:
        int(x[0])
    except:
        word_count_scaled = word_count_scaled + ((x[0], scale(x[1])),)

stop_words = re.findall(r'\w+', open(stop_words_file).read().lower())

for x in word_count_scaled[:300]:
    if x[0] in stop_words:
        n = word_count_scaled.index(x)
        # Drop index n from immutable tuple
        word_count_scaled = word_count_scaled[: n] + word_count_scaled[n + 1:]