Last active
February 24, 2018 13:44
-
-
Save seanbehan/6a71808079f297396d50 to your computer and use it in GitHub Desktop.
using generator comprehensions in python to extract words from epub files
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from glob import glob | |
from os.path import basename | |
from zipfile import ZipFile, is_zipfile | |
from re import findall | |
from nltk.corpus import words | |
ROOT_PATH = '/Volumes/USB20FD/downloads/*.epub' | |
WORDS = set(words.words()) | |
groups = (group for group in zip(*(iter(glob(ROOT_PATH)),)*100)) | |
zips = ((ZipFile(file) for file in group if is_zipfile(file)) for group in groups) | |
files = (((z.open(f.filename).read() for f in z.infolist() if '.html' in f.filename) for z in zip) for zip in zips) | |
texts = ((((WORDS & set(findall(r'[\w]+', t.lower()))) for t in txt) for txt in text) for text in files) | |
corpus = set() | |
for text in texts: | |
for txt in text: | |
for t in txt: | |
corpus = (corpus | t) | |
print (float(len(corpus)) / float(len(WORDS)))*100 | |
print "-"*75 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment