john-adeojo

# THIS FUNCTION PLOTS A BAR CHART


import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns


def bar(data, name):
    """

john-adeojo

import sentence_transformers
import torch
from sentence_transformers import SentenceTransformer

# Transform each clean text to 512 dimensional vectors (embeddings)
def embed(data):
    model = SentenceTransformer("distilbert-base-nli-mean-tokens")
    embeddings = model.encode(data, show_progress_bar=True)
    return embeddings

john-adeojo

import umap

# Peform dimensionality reduction on the embeddings
# helps improve the eprformance of HDBSCAN 
# high dimensionality is often handled poorly
def umap_embed(embeddings, n_neighbors):
    umap_embeddings = umap.UMAP(
        n_neighbors=n_neighbors, n_components=10, metric="cosine"
    ).fit_transform(embeddings)
    return umap_embeddings

john-adeojo

# Perform HDBSCAN clustering
import hdbscan

def clusters(umap_embeddings, n_neighbors):
    cluster = hdbscan.HDBSCAN(
        min_cluster_size=n_neighbors, metric="euclidean", cluster_selection_method="eom"
    ).fit(umap_embeddings)
    return cluster

john-adeojo

import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns

# Prepare data: Reduce dimensionality to 2 dimensions so we can visulaise the clusters
def visualise_2d(embeddings, n_neighbors):
    umap_data = umap.UMAP(
        n_neighbors=n_neighbors, n_components=2, min_dist=0.0, metric="cosine"
    ).fit_transform(embeddings)
    result = pd.DataFrame(umap_data, columns=["x", "y"])

john-adeojo

# Lets attach labels to these clusters to get our topics

def docs_per_topics(data):
    # Read text body list into dataframe
    docs_df = pd.DataFrame(data, columns=["Doc"])
    # bring in cluster labels
    docs_df["Topic"] = cluster.labels_
    # Aggregate text body by cluster
    docs_df["Doc_ID"] = range(len(docs_df))
    docs_per_topic = docs_df.groupby(["Topic"], as_index=False).agg({"Doc": " ".join})

john-adeojo

def mergetopics(docs_per_topic, topic_sizes):
    
    # Bring it all together to get a summary of topics, all text bodies within the topic and top size.
    docs_per_topic["Topic_words"] = docs_per_topic.Topic.map(top_n_words)
    reddit_topics = pd.merge(docs_per_topic, topic_sizes, how="left", on="Topic")
    
    return reddit_topics

john-adeojo

import transformers
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer

# Define funtion for summariser.
# Using Google's PEGASUS 


def summarise_topics(reddit_topics):

    # Initialise model and tokenizer

john-adeojo

posts_28th = top_posts.loc[top_posts.date == "2021-01-28"].copy()
data = list(posts_28th.body_clean)

# Step 1: Vectorise text-body to get embeddings
embeddings = embed(data)

# Step 2: Reduce dimensionality of the embeddings
n_neighbors = 3
umap_embeddings = umap_embed(embeddings, n_neighbors)

john-adeojo

Topic,	Topic_words	Size,	Summarised
-1	[('started', 0.00781), ('hfts', 0.00551), ('rh', 0.00513), ('apes', 0.0051), ('subreddit', 0.00497)]	38	["reddit is going to be filled with propaganda and nothing else after gme is over. i'll miss you guys and all the great times and sarcasm."]
1	[('republic', 0.04417), ('halted', 0.02762), ('participate', 0.02574), ('trade', 0.02448), ('stability', 0.02425)]	5	['**:- Trade Republic halted trading for a few hours yesterday due to "unforeseen load limits" for certain stocks, but has now lifted the restriction.']
3	[('escalade', 0.02442), ('schwab', 0.01865), ('john', 0.01819), ('000', 0.01504), ('cadillac', 0.01466)]	6	["i'm an idiot who didn't buy in to the $GME short squeeze and now i'm an idiot who is refusing to sell."]
4	[('crack', 0.05263), ('rocks', 0.04112), ('rock', 0.03993), ('st', 0.03594), ('gangstop', 0.01916)]	7	['shorted GameStop, shorted a bunch of crack rocks, shorted a bunch of crack rocks, shorted a bunch of crack rocks, shorted a bunch of crack rocks,