text stats

gistfile1.txt

text = TokenizedText

def textStats(text):
	length = 0
	punctuation = 0
	smallWordsCount = 0
	for word in text:
		if(word == "," or word == "." or word == ";" or word == ":"):
			punctuation += 1
		else:
			wordLength = len(word)
			length += wordLength
			if(wordLength <= 3):
				smallWordsCount += 1
	return {'avgWordLength':length/(len(text)-punctuation),'numWords':len(text), 'punctNum':punctuation, 'smallWordCound':smallWordsCount, 'phraseAvgSize':(len(text)/punctuation)}