Identify themes within list elements, then build common themes across list elements

build_themes.R

library(openai)
library(tidyverse)

identify_themes <- function(raw_text, summary_length = 1000, number_of_themes){
  
  response <- create_completion(
    model = "text-davinci-003",
    max_tokens = summary_length,
    prompt = paste("Find the", number_of_themes, "most common themes in the following text:", raw_text)
  )
  
  summary <- response$choices |>
    as_tibble() |>
    pull(text) |>
    str_trim(side = "both")
  
  return(summary)
  
}

summarize_lists <- function(raw_text, summary_length = 1000){
  
  response <- create_completion(
    model = "text-davinci-003",
    max_tokens = summary_length,
    prompt = paste("Organize the following lists and tell me what themes they have in common:", raw_text)
  )
  
  summary <- response$choices |>
    as_tibble() |>
    pull(text) |>
    str_trim(side = "both")
  
  return(summary)
  
}

file_list <- list.files(here::here(), pattern = "*.txt")
my_list <- lapply(file_list, function(x) read_lines(file.path(here::here(), x)))

build_themes <- function(list, n_themes){
  
  themes <- list()
  
  for(i in 1:length(list)){
    themes[[i]] <- identify_themes(list[i], number_of_themes = n_themes)
  }
  
  summary <- summarize_lists(paste(themes, collapse=""))
  
  return(summary)

}

build_themes(my_list, n_themes = 5)