audhiaprilliant

# HTTP library for Python
import requests

# Regular expression
import re

# Array manipulation
import collections

# Data manipulation

audhiaprilliant

# List of URL
urls = [
    'https://www.gutenberg.org/files/1661/1661-0.txt',
    'https://www.gutenberg.org/files/2701/2701-0.txt',
    'https://www.gutenberg.org/files/11/11-0.txt',
    'https://www.gutenberg.org/files/98/98-0.txt',
    'https://www.gutenberg.org/files/74/74-0.txt'
]

# Text

audhiaprilliant

# Lower text
def lowerCase(text):
    return text.lower()

# Numbers removal
def numberRemoval(text):
    return re.sub(
        pattern = '\d',
        repl = ' ',
        string = text

audhiaprilliant

# How to get a list of top words
def getTopWords(
    text: str
    ):
    # Split text by its whitespace
    list_words = text.split()
    # Count the word frequencies
    word_freq = collections.Counter(list_words)
    # Get top n words that have highest frequencies
    top_words = word_freq.most_common()

audhiaprilliant

# List of data
l_data = []

# Highest frequency
max_freq = top_words[0][1]

# Alpha
alpha = 1

# Loop

audhiaprilliant

# Data viz
plotnine.options.figure_size = (10, 4.8)
(
    ggplot(
        data = df[:20]
    )+
    geom_bar(
        aes(
            x = 'word',
            y = 'actual_freq'

audhiaprilliant

# Data viz
plotnine.options.figure_size = (10, 4.8)
(
    ggplot(
        data = df
    )+
    geom_line(
        aes(
            x = 'rank',
            y = 'zipf_freq',

audhiaprilliant

# Linear regression for ideal Zipf's line
linear = LinearRegression()
linear.fit(
    X = np.log(np.array(df['rank'])).reshape(-1, 1),
    y = np.log(df['zipf_freq'])
)
# Print slope and intercept
print('Intercept: {intercept}\nSlope: {slope}'.format(
        intercept = linear.intercept_,
        slope = linear.coef_[0]

audhiaprilliant

audhiaprilliant

# Data frame manipulation
import pandas as pd

# Mathematical operations
import numpy as np

# Data visualization
import plotnine
from plotnine import *