prune.py

#sg
#This script iterates over documents in a directory and strips tags
#Thanks to http://stackoverflow.com/questions/753052/strip-html-from-strings-in-python

from HTMLParser import HTMLParser

class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

def strip_tags(html):
    s = MLStripper()
    s.feed(html)
    return s.get_data()


#iterate over the files in the directory
def process(ipdir, opdir):
    for doc in os.listdir(ipdir):
        print 'processing ' + doc
        with open (ipdir + doc, "r") as myfile:
            data=myfile.read().replace('\n', '')
            data = strip_tags(data)
        with open(opdir + doc, "w") as myfile:
            myfile.write(data)

import os,sys
if __name__ == '__main__':
    ipdir = sys.argv[1]
    opdir = sys.argv[2]
    print ipdir, opdir
    process(ipdir, opdir)