How to Automatically Build Stopwords

zipfs_law_autostopwords_8.py

# Linear regression for ideal Zipf's line
linear = LinearRegression()
linear.fit(
    X = np.log(np.array(df['rank'])).reshape(-1, 1),
    y = np.log(df['zipf_freq'])
)
# Print slope and intercept
print('Intercept: {intercept}\nSlope: {slope}'.format(
        intercept = linear.intercept_,
        slope = linear.coef_[0]
    )
)

# Data viz
plotnine.options.figure_size = (10, 4.8)
(
    ggplot(
        data = df
    )+
    geom_line(
        aes(
            x = np.log(df['rank']),
            y = np.log(df['actual_freq']),
            group = 1
        ),
        size = 1.5
    )+
    geom_abline(
        intercept = linear.intercept_,
        slope = linear.coef_[0],
        size = 1,
        color = '#981220'
    )+
    labs(
        title = 'Zipf Distribution in English Literature'
    )+
    xlab(
        xlab = 'Rank (log)'
    )+
    ylab(
        ylab = 'Frequency of words (log)'
    )+
    theme_minimal()
)