tmp.R

tmp.R

pacman::p_load(tidyverse)

source("https://gist.githubusercontent.com/benjaminguinaudeau/2db5647ac4691559ed0f5af6d67166ce/raw/72dd86762962fc6391df3a94c267e7a48ffb5712/get_accessible_videos.R")

reconnect <- function(){
  
  file <- ""
  trig <- try(stop(), silent = T)
  
  cli::cli_h1("[ {Sys.time()} ] Reconnecting")
  
  while(inherits(trig, "try-error")){
    if(file.exists(file)) file.remove(file)
    bashR::sudo("killall openvpn")
    file <- sample(stringr::str_subset(dir("/etc/openvpn", full.names = T), "\\.tcp"), 1)
    system(glue::glue("echo \"auth-user-pass /etc/openvpn/auth.txt\" >> {file}"))
    bashR::sudo(glue::glue("/usr/local/Cellar/openvpn/2.4.8/sbin/openvpn {file} > /etc/openvpn/log.txt &"))
    
    Sys.sleep(2)
    
    while(!any(str_detect(tail(readLines("/etc/openvpn/log.txt"), 3), "Initialization Sequence Completed"))){
      Sys.sleep(1)
    }
    trig <- try(httr::GET("http://google.com",  httr::timeout(5)), silent = T)
  }
  
  cli::cli_alert_success("[ Sys.time() ] {file}")
}

if(!fs::dir_exists("~/yt_n_videos")){fs::dir_create("~/yt_n_videos")}
yt_commentors <- read_rds("~/Downloads/yt_commentors_sample.rds")

reconnect()

yt_commentors %>%
  anti_join(map_dfr(dir("~/yt_n_videos", full.names = T), read_rds)) %>%
  pull(channel_id) %>%
  split(1:length(.) %/% 1000) %>% #bashR::simule_map(1)
  iwalk(~{
    
    cli::cli_h2("[ {Sys.time()} ] Batch {.y}")
    
    out <- .x %>%
      imap_dfr(~{
        if(as.numeric(.y) %% 100 == 0){ cli::cli_alert_info("{.y}") }
        a <- try({get_accessible_videos(.x)})

        if(inherits(a, "try-error")){ reconnect() ; return(tibble::tibble()) }

        return(a)
      })
    
    write_rds(out, here::here(glue::glue("yt_n_videos/{as.numeric(nanotime::nanotime(Sys.time()))}.rds")))
    
    # reconnect()
    # wait <- sample(1, 60:70)
    # cli::cli_alert_danger("[ {Sys.time()} ] Waiting {wait} seconds")
    # Sys.sleep(wait)
  })