getDatafromGhostery

ghosteryScrape.r

require(rjson)
library(httr)
require(gtools)


dataframeFromJSON <- function(l) {
  l1 <- lapply(l, function(x) {
    x[sapply(x, is.null)] <- NA
    unlist(x)
  })
  keys <- unique(unlist(lapply(l1, names)))
  l2 <- lapply(l1, '[', keys)
  l3 <- lapply(l2, setNames, keys)
  res <- data.frame(do.call(rbind, l3))
  return(res)
}


########################get DATA
getGhosteryData <- function(BaseURL, country){
  #token=248624862  #248624862
  url <- "https://livescan.ghosteryenterprise.com/api/TrackerMapLiveApi/BuildLiveMap"
  body = list(LoginToken = 248624862, ScanCountryId = country, ScanProxy="", ScanURL = BaseURL)
  #ScanURL=bild.de&ScanProxy=&ScanCountryId=1&LoginToken=248624862
  
  r <- POST(url, body = body, encode = "form")
  
  if(http_status(r)$category == "success"){
    
    #co <-content(r, "parsed")
    js <- fromJSON(content(r, "text"))
    nodes <- js$NodeMap$nodes
    out<-dataframeFromJSON(nodes)
    
    return(out)
  }
  else{
    print(paste("error", BaseURL))
  }
}


findID <- function(txt){
  CountryCodes[CountryCodes$Code == txt, ]$id  
}


##Country Codes 
CountryCodes <- data.frame(Code=c("DE", "FR", "UK", "BE", "AT", "CH", "NL", "ES", "IT", "PL", "RU", "US"), id= c(3, 5, 6, 20, 19, 21, 8, 4, 7, 18, 13,1))

out_DE <- getGhosteryData("www.audi.de" , 1)
out_CH <- getGhosteryData("audi.ch" , findID("CH"))