us_visits.R

# r code for the animations at
# https://twitter.com/ResulUmit/status/1201779741155635200?s=20

# load libraries ----------------------------------------------------------

library(tidyverse)
library(lubridate)
library(fuzzyjoin)
library(rvest)
library(magrittr)
library(countrycode)
library(maps)
library(gganimate)

# note that the following packages must also be installed, but they don't
# need to be loaded: gifski, transformr


# get and tidy the data ---------------------------------------------------

datalist <- list()   

for(i in 1945:2018) {
    
    df.collect <- data.frame(text = read_html(paste0("https://history.state.gov/departmenthistory/visits/", i)) %>%
                              html_nodes("td") %>%
                              html_text())
    
    datalist[[i]] <- df.collect
  }


df_visits <- do.call(rbind, datalist) %>% 
  mutate(order = rep(c("v_pers", "v_cntry", "v_type", "v_time"), length(text)/4),
         id = rep(1:as.numeric(length(text)/4), each = 4)) %>% 
  spread(order, text) %>% 
  mutate(v_cntry = recode(v_cntry, "France (Provisional Government)" = "France",
                                   "Germany, Federal Republic of" = "Germany",
                                   "Israel (Provisional Government)" = "Israel",
                                   "Slovak Republic" = "Slovakia"),
         v_cntry = str_trim(v_cntry),
         v_type = str_to_lower(v_type),
         v_visit = case_when(str_detect(v_type, "met with president") |
                               str_detect(v_type, "meeting with president") |
                               str_detect(v_type, "official visit") |
                               str_detect(v_type, "working visit") |
                               str_detect(v_type, "presidential guest") |
                               str_detect(v_type, "state visit") ~ 1, TRUE ~ 0),
         v_time = str_replace(v_time, "–[^>]+,", ","),
         v_time = as.Date(parse_date_time(v_time, orders = c("ymd", "dmy", "mdy"))),
         cntry_code = countrycode(v_cntry, origin = "country.name", destination = "iso3c"))


# world map ---------------------------------------------------------------

yes_visits <- df_visits %>% 
  mutate(year = as.numeric(format(v_time, "%Y")),
         period = case_when(year < 1950 ~ "1945 - 1950",
                                   1950 <= year & year < 1960 ~ "1945 - 1960",
                                   1960 <= year & year < 1970 ~ "1945 - 1970",
                                   1970 <= year & year < 1980 ~ "1945 - 1980",
                                   1980 <= year & year < 1990 ~ "1945 - 1990",
                                   1990 <= year & year < 2000 ~ "1945 - 2000",
                                   2000 <= year & year < 2010 ~ "1945 - 2010",
                                   2010 <= year & year < 2019 ~ "1945 - 2018")) %>% 
  group_by(cntry_code, period) %>% 
  summarise(n_visits = n()) %>% 
  ungroup() %>% 
  complete(cntry_code, period, fill = list(n_visits = 0)) %>% 
  group_by(cntry_code) %>% arrange(period, .by_group = TRUE) %>% 
  mutate(cm_visits = cumsum(n_visits)) %>% 
  ungroup() %>% 
  select(-n_visits)

world_map <- map_data("world") %>%
  select(-subregion) %>% filter(region != "Antarctica") %>% 
  mutate(cntry_code = countrycode(region, origin = "country.name", destination = "iso3c"))

no_visits <- data.frame(cntry_code = setdiff(world_map$cntry_code, yes_visits$cntry_code),
                        period = c("1945 - 1950", "1945 - 1960", "1945 - 1970", "1945 - 1980", 
                                   "1945 - 1990", "1945 - 2000", "1945 - 2010", "1945 - 2018"),
                        cm_visits = 0) %>% 
  complete(cntry_code, period, fill = list(cm_visits = 0))


all_visits <- rbind(yes_visits, no_visits)

df <- left_join(world_map, all_visits, by = "cntry_code")

world_gif <- ggplot(df) +
  aes(x = long, y = lat, group = group, fill = cm_visits) +
  geom_polygon(colour = "grey60", size = 0.3) +
  theme_void() +
  theme(panel.grid=element_blank(),
        axis.ticks=element_blank(),
        panel.border=element_blank(),
        axis.text=element_blank(),
        legend.position=c(0.1, 0.20)) +
  scale_fill_gradient(name = "", low = "gray99", high = "gray0",
                      limits=c(0, 120),
                      breaks=c(0, 30, 60, 90, 120)) +
  labs(title = "Period: {closest_state}") +
  transition_states(period)

animate(world_gif, width = 600, height = 335)

anim_save("world_visits.gif")


# europe map --------------------------------------------------------------

df_eur <- df %>% 
  mutate(continent = countrycode(cntry_code, origin = "iso3c", destination = "continent")) %>% 
  filter(continent == "Europe" & cntry_code != "RUS")

eur_gif <- ggplot(df_eur) +
  aes(x = long, y = lat, group = group, fill = cm_visits) +
  geom_polygon(colour = "grey60", size = 0.3) +
  theme_void() +
  theme(panel.grid=element_blank(),
        axis.ticks=element_blank(),
        panel.border=element_blank(),
        axis.text=element_blank(),
        legend.position=c(0.1, 0.20)) +
  scale_fill_gradient(name = "", low = "gray99", high = "gray0",
                      limits=c(0, 120),
                      breaks=c(0, 30, 60, 90, 120)) +
  labs(title = "Period: {closest_state}") +
  transition_states(period)

animate(eur_gif, width = 600, height = 335)

anim_save("europe_visits.gif")


# bar gif -----------------------------------------------------------------

df_rank <- df %>% 
  select(cntry_code, period, cm_visits) %>% 
  distinct() %>% 
  group_by(period) %>% 
  mutate(rank = rank(-cm_visits, ties.method = "first")) %>% 
  filter(rank <= 10) %>% 
  ungroup()

df_bars <- left_join(df, df_rank, by = "cntry_code")

bar_gif <- ggplot(df_rank, aes(x = rank, y = cm_visits, group = cntry_code, fill = cntry_code)) +
  geom_bar(stat = "identity", aes(fill = cntry_code), alpha = 0.8, color = NA) +
  geom_text(aes(y = 0, label = paste(cntry_code, " ")), vjust = 0.2, hjust = 1) +
  coord_flip() + 
  theme_minimal() + 
  theme(axis.text.y = element_blank(),
        axis.ticks.y = element_blank()) +
  scale_y_continuous(limits = c(-10, 120),
                     breaks = c(0, 30, 60, 90, 120),
                     labels= c(0, 30, 60, 90, 120)) +
  scale_x_reverse() +
  guides(color = FALSE, fill = FALSE) +
  labs(title = "Period: {closest_state}", y = "", x = "", fill = "") +
  transition_states(period)

animate(bar_gif, width = 600, height = 335)

anim_save("bar_visits.gif")