custom-pipes-spacy.py

def lemmatizer(doc):
    """
    This takes in a doc of tokens from the NER and lemmatizes them. 
    Pronouns (like "I" and "you" get lemmatized to '-PRON-', so I'm removing those.
    """
    doc = [token.lemma_ for token in doc if token.lemma_ != '-PRON-']
    doc = u' '.join(doc)
    return nlp.make_doc(doc)
    
def remove_stopwords(doc):
    """
    This will remove stopwords and punctuation.
    Use token.text to return strings, which we'll need for Gensim.
    """
    doc = [token.text for token in doc if token.is_stop != True and token.is_punct != True and token.text.strip() != '' and token.is_digit != True]
    return doc