analysis.py

from operator import itemgetter
from collections import Counter

flat_list = [i for sublist in filtered_tokens for i in sublist]

# Count how many times each word appears
count = Counter(flat_list).items()
sorted_count = sorted(count, key = itemgetter(1))
sorted_count.reverse()

# Select 5000 most frequent words
top5000 = [i[0] for i in sorted_count[:5000]]