fareshare-food-type-cleanups.py

import re
from spellchecker import SpellChecker
import csv

food_col = []
food_output = []
errors = []
line = 0

with open('Data/survey.csv', 'r') as csvfile:
    reader = csv.DictReader(csvfile, delimiter=',', quotechar='"')

    for row in reader:
        food_col.append(row['top_5_food_types'])


spell = SpellChecker()

early_replace = {
    ' free ': '-free-',
    ' free': '-free',
    ' bar': '-bar',
    ' alternative': '-alternative',
    'non ': 'non-',
    ' oil': '-oil',
    'semi ': 'semi-',
    'soft ': 'soft-',
    ' supplement': '-supplement',
    'sea food': 'seafood',
    'sea-food': 'seafood',
    'milk powder': 'powdered-milk',
    ' soup': '-soup',
    'u.h.t': 'uht',
    'sugar free ': 'sugar-free-',
    'sugar-free ': 'sugar-free-',
    'spicy ': 'spicy-',
    'savoury ': 'savoury-',
    'processed ': 'processed-',
    'tinned food': 'tinned-food',
    'canned food': 'tinned-food',
    ' carton': '-carton',
    ' cordial': '-cordial',
    'reduced fat ': 'reduced-fat-'
}

spellcheck_ignore = [
'uht'
]

switch = {
    'product': 'produce',
    'fridge': 'chilled',
    'desert': 'dessert',
    'choc': 'chocolate',
    'sauce': 'condiment',
    'sweet': 'confectionary',
    'marge': 'margarine',
    'marg': 'margarine',
    'yogurt': 'yoghurt',
    'cold': 'chilled',
    'tin': 'tinned',
    'can': 'tinned',
    'carb': 'carbohydrate',
    'filler': 'filling',
    'lam': 'lamb',
    'veg': 'vegetable',
    'uhtmilk': 'uht-milk',
    'long life milk': 'uht-milk',
    'long-life milk': 'uht-milk',
    'long-life-milk': 'uht-milk',
    'uncooked': 'raw',
    'pack': 'packet',
    'serial': 'cereal',
    'source': 'sauce',
    'tom': 'tomato',
    'tub': 'container',
    'box': 'container',
    'sea food': 'seafood',
    'sea-food': 'seafood',
    'milk powder': 'powdered-milk',
    'raw': 'fresh',
    'dry': 'dried',
    'chiller': 'chilled',
    'carbonated': 'fizzy',
    

}

stop_words = [
    "a", "about", "above", "above", "across", "after", "afterwards", "again", "against", "all", "almost", "alone", "along", "already", "also","although","always","am","among", "amongst", "amoungst", "amount",  "an", "and", "another", "any","anyhow","anyone","anything","anyway", "anywhere", "are", "around", "as",  "at", "back","be","became", "because","become","becomes", "becoming", "been", "before", "beforehand", "behind", "being", "below", "beside", "besides", "between", "beyond", "bill", "both", "bottom","but", "by", "call", "can", "cannot", "cant", "co", "con", "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", "down", "due", "during", "each", "eg", "eight", "either", "eleven","else", "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone", "everything", "everywhere", "except", "few", "fifteen", "fify", "fill", "find", "fire", "first", "five", "for", "former", "formerly", "forty", "found", "four", "from", "front", "full", "further", "get", "give", "go", "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his", "how", "however", "hundred", "ie", "if", "in", "inc", "indeed", "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter", "latterly", "least", "less", "ltd", "made", "many", "may", "me", "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly", "move", "much", "must", "my", "myself", "name", "namely", "neither", "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone", "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on", "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our", "ours", "ourselves", "out", "over", "own","part", "per", "perhaps", "please", "put", "rather", "re", "same", "see", "seem", "seemed", "seeming", "seems", "serious", "several", "she", "should", "show", "side", "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone", "something", "sometime", "sometimes", "somewhere", "still", "such", "system", "take", "ten", "than", "that", "the", "their", "them", "themselves", "then", "thence", "there", "thereafter", "thereby", "therefore", "therein", "thereupon", "these", "they", "thick", "thin", "third", "this", "those", "though", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "top", "toward", "towards", "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us", "very", "via", "was", "we", "well", "were", "what", "whatever", "when", "whence", "whenever", "where", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "whoever", "whole", "whom", "whose", "why", "will", "with", "within", "without", "would", "yet", "you", "your", "yours", "yourself", "yourselves", "the"
]

extras = [
    'able', 'accompanying', 'actually', 'added', 'ambient', 'good', 'item', 'big', 'thank', 'with', 'cupboard', 'etc', 'ect', 'cutlery', 'scratch',
    'prepare', 'depend', 'week', 'various', 'wholesome', 'including', 'na', 'eg',
    'fareshare', 'supply', 'butcher', 'carb', 'like', 'change', 'periodically', 'dependent', 'availability', 'donation', 'provider', 'cooking',
    'particular', 'part', 'particularly', 'need', 'not', 'order', 'price', 'rely', 'require', 'requirement', 'satisfied', 'scout', 'scratch', 'store', 'specific',
    'such', 'suitable', 'surplus', 'thing', 'supply', 'there', 'unusual', 'used', 'useful', 'use', 'unused', 'varied', 'vary', 'top', 'tool', 'with', 'very', 'versatile', 'trip',
    'standard', 'such', 'staple', 'so', 'space', 'send', 'service', 'severe', 'ppl', 'problem', 'range', 'mother', 'eg', 'type', 'e.g.', 'i.e.', 'depend', 'easy',
    'simple', 'fast', 'slow', 'variety', 'usual', 'style', 'staple', 'stuff', 'requirement', 'reception', 'receive', 'quick', 'possible', 'owning', 'autistic',
    "autism", "green", "blue", "red", "yellow", "healthy", "children", "just", "work", "ymca", "takeaway", "ingredient", "individual", "intolerant", "item", "important",
    "ideally", "help", "form", "incl"
]

pairs = [
'baked beans',
'sandwich filler',
'sandwich meat',
'sausage roll',
'meat pie',
'pork pie',
'non perishable good',
'processed food',
'cocktail sausage',
'ready meal',
'semi skimmed milk',
'skimmed milk',
'double cream',
'fruit juice',
'orange juice',
'short bread',
'milk shake',
'root vegetable',
'wholemeal bread',
'whole milk',
'snack food',
'red sauce',
'brown sauce'
'self raising flour',
'self-raising flour',
'fresh produce',
' free ',
' free',
' bar',
' alternative',
'non ',
' oil',
'semi ',
'soft ',
' supplement',
' soup',
'sugar free ',
'sugar-free ',
'spicy ',
'savoury ',
'processed ',
'picnic food',
'fruit pouch',
'dairy produce',
'fizzy ',
'sandwich filling',
'low fat ',
'vegetable oil',
'hard ',
'soft ',
'garlic bread',
'washing up liquid',
'washing-up liquid',
'toilet roll',
'savoury '
]


styles = ['cooked', 'cold', 'fresh', 'bag', 'sliced', 'prepared', 'raw', 'dried', 'tinned', 'frozen', 'packet', 'beverage', 'hot', 'bottled', 'packet', 'mouldy', 'chilled', 'long']
dietaries = ['vegan', 'vegetarian', 'halal', 'unprocessed']
size = ['large', 'small']

post_replaces = {
'fresh-': '',
'dried-rice': 'rice',
'dried-pasta': 'pasta'

}


stop_words.extend(extras)

def clean_entry(s):
    s = s.lower().strip()
    for kpair, vpair in early_replace.items():
        s = s.replace(kpair, vpair)

    s_split = s.split()
    tout = ''
    new_split = []

    for word in s_split:
        if word not in stop_words:
            word = word.strip('.')
            word = re.sub('\s*-\s*', '-', word)
            word = re.sub('[^\-a-z]', '', word)
            word = re.sub('ies$', 'y', word)
            word = re.sub('s$', '', word)
            if word not in spellcheck_ignore:
                misspelled = spell.unknown([word])
                for word in misspelled:
                    corr =spell.correction(word)
                    if corr is not None and corr != 'a':
                        word = corr

            if word not in stop_words:
                sw = switch.get(word, None)
                if sw is not None:
                    word = sw
                if word is not None and word != '' and word != '-':
                    new_split.append(word)

    tout = ' '.join(new_split)
    for pair in pairs:
        tout = tout.replace(pair, pair.replace(' ', '-'))
    for style in styles:
        tout = tout.replace(style + ' ', style + '-')
    for dietary in dietaries:
        tout = tout.replace(dietary + ' ', dietary + '-')

    for post_replace, newp in post_replaces.items():
        tout = tout.replace(post_replace, newp)


    return tout.strip()

def do_split(t):

    return re.split('(\\r|\\n|\/|^a\s|\sa\s|;|,|\d+|\/\d?|\.\.)', t)

top_level = []
foods_split_temp = food_col

print("Split entries into foods")
print('---------------------------------')

for food_cell in food_col:
    foods_split = []

    split_entry = do_split(food_cell)
    split_entry = [x for x in split_entry if x is not None and x != "\n" and x != '' and len(x) > 0]

    for s in split_entry:
        combined_item = clean_entry(s)

        cells = combined_item.split(' ')
        foods_split.extend(cells)
        food_output.extend(cells)

    foods_split = [x for x in foods_split if x is not None and x != "\n" and x != '' and len(x) > 0]
    foods_split.extend(['','','','',''])
    # print("orig:",food_cell)
    print(','.join(foods_split[0:5]))

    error_res = ''
    if '-' in foods_split:
        error_res = 'ERROR'
    errors.append(error_res)

    line += 1

print("Results")
print('---------------------------------')
food_outputset = sorted(set(food_output))
print('\n'.join(food_outputset))

food_count = {}

for k in food_output:
    res = food_count.get(k, None)
    if res is None:
        food_count[k] = 1
    else:
        food_count[k] = res + 1

for k, v in food_count.items():
    print(k+','+str(v))