Identify keywords in text

plagiarism_checker.py

from sumy.parsers.plaintext import PlaintextParser
from sumy.nlp.tokenizers import Tokenizer
from sumy.summarizers.lsa import LsaSummarizer as Summarizer
from sumy.nlp.stemmers import Stemmer
from sumy.utils import get_stop_words

LANG = "english"
tokenizer = Tokenizer(LANG)
stemmer = Stemmer(LANG)
summarizer = Summarizer(stemmer)
summarizer.stop_words = get_stop_words(LANG)


def LSA(text):
    parser = PlaintextParser.from_string(text, tokenizer)
    sentences = list([str(s) for s in summarizer(parser.document, 10)])
    return ' '.join(sentences)


from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords

sw = set(stopwords.words(LANG))


def process(text):
    summarized = LSA(text)
    tokenized = word_tokenize(summarized)
    filtered = [word for word in tokenized if word not in sw]
    return filtered

Usage in plagiarism checker:

text = ... # load string text from somewhere, regex out special chars
keywords = process(text)
# search google using google search api for keywords and rate matches using ssim