upshot_certainty_gap.R

remove(list=ls())
require(jsonlite)
require(ggplot2)
require(reshape2)
require(scales)

jr=theme(panel.background =  element_rect(fill = NA, colour = "black", size = 0.25),
         panel.border =      element_blank(),
         panel.grid.major =  element_line(colour = "black", size = 0.05),
         panel.grid.minor =  element_line(colour = "black", size = 0.05),
         plot.title=element_text(size=22, family="Helvetica Neue Light"),
         axis.title.x=element_text(size=18, family="Helvetica Neue Light"),
         axis.text.x=element_text(colour="black", size=18, family="Helvetica Neue Light"),
         axis.title.y=element_text(size=18, family="Helvetica Neue Light"),
         axis.text.y=element_text(colour="black",size=18, family="Helvetica Neue Light"),
         strip.text.x = element_text(size = 22,family="Helvetica Neue Light"),
         strip.text.y = element_text(size = 22,family="Helvetica Neue Light"),
         legend.title = element_text(size=18, family="Helvetica",face="bold"),
         legend.text = element_text(size=18, family="Helvetica Neue Light"),
         strip.background = element_rect(colour = "grey", fill = "white")) 

page_ids=jsonlite::fromJSON('https://int.nyt.com/newsgraphics/2018/live-polls-2018/all-races.json')$page_id

certain_disparity_summary=plyr::rbind.fill(lapply(gsub('elections-poll-','',page_ids),function(j){
  
  message(j)
    hm=jsonlite::fromJSON(paste0('https://int.nyt.com/newsgraphics/2018/live-polls-2018/races/elections-poll-',j,'.json'))$results
  hm=plyr::rbind.fill(lapply(names(hm),function(x){
    
    data.frame(hm[[x]])
    
  }))
  
  hm$race=j
  hm$isIncumbent=jsonlite::fromJSON(paste0('https://int.nyt.com/newsgraphics/2018/live-polls-2018/races/elections-poll-',j,'.json'))$isIncumbent
  
  return(hm)
  
}))

certain_disparity_summary$in_progress='Finished'
certain_disparity_summary$in_progress[certain_disparity_summary$n<400]='In Progress'
certain_disparity_summary$race=toupper(substr(certain_disparity_summary$race,1,4))

certain_disparity_summary$margin[certain_disparity_summary$isIncumbent=='rep']=(-certain_disparity_summary$margin[certain_disparity_summary$isIncumbent=='rep'])
certain_disparity_summary$margin[certain_disparity_summary$race=='NC09']=-certain_disparity_summary$margin[certain_disparity_summary$race=='NC09']
certain_disparity_summary$margin[certain_disparity_summary$race=='TNSE']=-certain_disparity_summary$margin[certain_disparity_summary$race=='TNSE']

comp=reshape2::dcast(certain_disparity_summary[certain_disparity_summary$key %in% c('certain_only','final'),],race+in_progress~key,
                value.var='margin')

ggplot(comp,aes(certain_only-final)) +
  geom_density(aes(fill=in_progress), alpha=.2,col='white') +
  jr + theme(axis.text.y = element_blank()) +
  ylab('') + scale_x_continuous(labels=percent,'Dem. margin among certain to vote respondents vs. "final" estimate') + 
  geom_vline(xintercept = 0,linetype=2) +
  scale_fill_manual('',values=c('black','darkorchid4')) +
  ggtitle('Density distributions of certain vs. "final" estimate','From NYT Upshot/Siena polls')