acrymble

# Given a URL, return string of lowercase text from page.

def webPageToText(url):
    import urllib2
    response = urllib2.urlopen(url)
    html = response.read()
    text = stripTags(html).lower()
    return text

acrymble

# Given name of calling program, a url and a string to wrap,
# output string in html body with basic metadata
# and open in Firefox tab.

def wrapStringInHTML(program, url, body):
    import datetime
    from webbrowser import open_new_tab
    now = datetime.datetime.today().strftime("%Y%m%d-%H%M%S")
    filename = program + '.html'
    f = open(filename,'w')

acrymble

# html-to-freq-3.py
 
import obo

# create sorted dictionary of word-frequency pairs
url = 'http://www.oldbaileyonline.org/print.jsp?div=t17800628-33'
text = obo.webPageToText(url)
fullwordlist = obo.stripNonAlphaNum(text)
wordlist = obo.removeStopwords(fullwordlist, obo.stopwords)
dictionary = obo.wordListToFreqDict(wordlist)

acrymble

wordstring = 'it was the best of times it was the worst of times '
wordstring += 'it was the age of wisdom it was the age of foolishness'
wordlist = wordstring.split()
print wordlist[0:4]
-> ['it', 'was', 'the', 'best']

print wordlist[0:6]
-> ['it', 'was', 'the', 'best', 'of', 'times']

print wordlist[6:10]

acrymble

i = 0
for items in wordlist:
    print wordlist[i: i+5]
    i += 1

acrymble

# Given a list of words and a number n, return a list
# of n-grams.

def getNGrams(wordlist, n):
    return [wordlist[i:i+n] for i in range(len(wordlist)-(n-1))]

acrymble

#useGetNGrams.py

import obo

wordstring = 'it was the best of times it was the worst of times '
wordstring += 'it was the age of wisdom it was the age of foolishness'
allMyWords = wordstring.split()

print obo.getNGrams(allMyWords, 5)

acrymble

test1 = 'here are four words'
test2 = 'this test sentence has eight words in it'
getNGrams(test1.split(), 5)
-> []

getNGrams(test2.split(), 5)
-> [['this', 'test', 'sentence', 'has', 'eight'], 
['test', 'sentence', 'has', 'eight', 'words'], 
['sentence', 'has', 'eight', 'words', 'in'], 
['has', 'eight', 'words', 'in', 'it']]

acrymble

print (7 // 2)
print (5 // 2)
print (3 // 2)

acrymble

# Given a list of n-grams, return a dictionary of KWICs,
# indexed by keyword.

def nGramsToKWICDict(ngrams):
    kwicdict = {}
    keyindex = len(ngrams[0]) // 2