Text analysis with word frequency, topic modeling using LDA, and word clouds. https://www.coursera.org/learn/applying-data-analytics-business-in-marketing/home/week/3

text-analysis.R

# Load the rvest package
library(rvest)
library(ggplot2)
library(tidytext)
library(tidyverse)
library(topicmodels)
library(tm)
library(wordcloud)
library(RColorBrewer)

# Define the function
extract_posts <- function() {
  # Set the user_agent for web requests.
  httr::set_config(httr::user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"))
  
  # Read the html structure of the web page
  html <- read_html("https://bogleheads.org/")
  
  # Create an empty list to store the results
  results <- list()
  
  # Find the posts_table element
  posts_table <- html_node(html, "#posts_table")
  
  # Find all the td elements with the style "vertical-align:baseline;" within the posts_table
  posts <- html_nodes(posts_table, "tr[style='vertical-align:baseline;']")
  
  # Loop through each post element
  for (i in 1:length(posts)) {
    # Extract the post url and title from the third td element
    post_url <- html_attr(html_node(posts[i], "td:nth-child(3) a"), "href")
    post_title <- html_text(html_node(posts[i], "td:nth-child(3) a"))
    
    # Extract the post author from the next td element with the style "white-space:nowrap;"
    post_author <- html_text(html_node(posts[i], "td[style='white-space:nowrap;']"))
    
    # Trim leading and trailing whitespace from the values
    post_url <- trimws(post_url)
    post_title <- trimws(post_title)
    post_author <- trimws(post_author)
    
    # Add the extracted information to the results list
    results[[i]] <- list(
      url = post_url,
      title = post_title,
      author = post_author
    )
  }
  
  # Return the results list
  return(results)
}

data <- extract_posts()

# Create a dataframe.
df <- do.call(rbind, lapply(data, function(x) {
  data.frame(matrix(unlist(x), nrow=length(x), byrow=T))
}))
df$author <- as.factor(df$author)

# Add a row number to the data.
df$row_num <- seq_len(nrow(df))

# Tokenize and normalize (lowercase, remove punctuation).
df_tokens <- unnest_tokens(df, word, title, token = 'words')

# Remove stop words.
df_tokens <- filter(anti_join(df_tokens, stop_words, by = 'word'), nchar(word) >= 3)

# Top 20 most frequent tokens.
top20 <- head(arrange(count(df_tokens, word), desc(n)), 20)

# Draw a chart of the top 20 tokens.
ggplot(
  data = top20,
  aes(x = n, y = reorder(word, n))) +
  geom_col() +
  theme_classic() + 
  xlab('Number of occurrences per token') +
  ylab('Token')

# Create a document term matrix.
dtm <- cast_dtm(count(df_tokens, row_num, word), row_num, word, n)

lda <- LDA(dtm, k = 3, control = list(seed = 12))

# Dispay per-topic-per-wprd probabilities (beta is the probability of each word within a topic).
topics <- tidy(lda, matrix = 'beta')

top_terms <- arrange(ungroup(slice_max(group_by(topics, topic), beta, n = 20)), topic, -beta)

# Plot the top 20 terms.
data <- mutate(top_terms, term = reorder_within(term, beta, topic))
data$topic <- as.factor(data$topic)

# Display the first 3 topics and the words associated with them according to probability within the topic.
ggplot(data = data, aes(beta, term, fill = data$topic)) +
  geom_col(show.legend = F) +
  theme_minimal() + 
  ggtitle('Top terms by topic') +
  facet_wrap('topic', scales = 'free') +
  scale_y_reordered()

# Display a word cloud of the terms.
text <- top_terms$term

# Create a text corpus
corpus <- Corpus(VectorSource(text))

# Convert the text to lower case
corpus <- tm_map(corpus, content_transformer(tolower))

# Remove common stopwords
corpus <- tm_map(corpus, removeWords, stopwords("english"))

# Create a term-document matrix
tdm <- TermDocumentMatrix(corpus)

# Convert the term-document matrix to a matrix
m <- as.matrix(tdm)

# Get word counts
word_freqs <- sort(rowSums(m), decreasing=TRUE) 

# Create a data frame with words and their frequencies
df <- data.frame(word=names(word_freqs), freq=word_freqs)

# Create the word cloud
wordcloud(words = df$word, freq = df$freq, min.freq = 1,
          max.words=200, random.order=FALSE, rot.per=0.35, 
          colors=brewer.pal(8, "Dark2"))

Word cloud of terms

Top 20 terms

Topics