Get the most frequent words in a pandas text column

word_counter.py

from collections import Counter
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from tqdm import tqdm 
import pandas as pd
import string

# Download NLTK stopwords
import nltk
nltk.download('stopwords')

tqdm.pandas()

# Assuming 'df' is your DataFrame with a 'title' column
# If not, replace 'df' with your actual DataFrame name

# Function to filter out unwanted words
def is_valid_word(word):
    return word.isalpha() and len(word) >= 3

# Tokenize the text column into individual words and filter
df['tokenized_title'] = df['title'].astype(str).progress_apply(word_tokenize)
df['tokenized_title'] = df['tokenized_title'].apply(lambda tokens: [word.lower() for word in tokens if is_valid_word(word)])

# Flatten the list of lists into a single list of words
all_words = [word for sublist in df['tokenized_title'] for word in sublist]

# Convert stop_words to a set for faster membership tests
stop_words_set = set(stopwords.words('english'))

# Use set operations for efficient filtering
filtered_words = [word for word in all_words if word not in stop_words_set]

# Count the frequency of each word
word_counts = Counter(filtered_words)

# Select the top 100 most frequent words
top_words = dict(word_counts.most_common(100))

# Create a new DataFrame with the selected words and their frequencies
top_words_df = pd.DataFrame(list(top_words.items()), columns=['Word', 'Frequency'])
top_words_df