How to Automatically Build Stopwords

zipfs_law_autostopwords_3.py

# Lower text
def lowerCase(text):
    return text.lower()

# Numbers removal
def numberRemoval(text):
    return re.sub(
        pattern = '\d',
        repl = ' ',
        string = text
    )

# Non-alphanumeric removal
def nonAlphanumericRemoval(text):
    return re.sub(
        pattern = '[^a-zA-Z\d]',
        repl = ' ',
        string = text
    )

# Whitespaces removal
def whitespacesRemoval(text):
    # Returns string with leading whitespaces removed
    x = str(text).lstrip()
    # Returns string with trailing whitespaces removed
    x = str(x).rstrip()
    # Substitute multiple whitespaces with single whitespace
    x = re.sub(
        pattern = ' +',
        repl = ' ',
        string = str(x)
    )
    return x

# Compile text preprocessing into one function
def textPreprocessing(text):
    # Lower text
    x = lowerCase(text)
    # Numbers removal
    x = numberRemoval(x)
    # Non-alphanumeric removal
    x = nonAlphanumericRemoval(x)
    # Whitespaces removal
    x = whitespacesRemoval(x)
    return x
  
# Perform simple text preprocessing
text_clean = textPreprocessing(
    text = text
)