gistfile1.py

import logging
logging.basicConfig(format='%(asctime)s : %(Levelname)s : %(message)s', level=logging.INFO)

from gensim import corpora, models, similarities

stoplist = set('method,device,sysetem,apparatus,a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your'.split(','))

dictionary = corpora.Dictionary(line.lower().split() for line in open('abstracts.txt'))
stop_ids = [dictionary.token2id[stopword] for stopword in stoplist if stopword in dictionary.token2id]
once_ids = [tokenid for tokenid, docfreq in dictionary.dfs.iteritems() if docfreq == 1]
dictionary.filter_tokens(stop_ids + once_ids)
dictionary.compactify()

class MyCorpus(object):
	def __iter__(self):
		for line in open('abstracts.txt'):
			yield dictionary.doc2bow(line.lower().split())

corpus = MyCorpus()

model = ldamodel.LdaModel(corpus, id2word=dictionary, num_topics=100, distributed=TRUE)