api_scrape.R

library(httr)
library(jsonlite)
library(DBI)
library(purrr)

test_url <-
  paste0(
    "http://opendata-ajuntament.barcelona.cat/data/api/3/action/resource_search?query=name:",
    "bicing"
  )

# Wrap GET so that whenever there's a 404 it returns an R error
my_GET <- function(x, config = list(), ...) {
  stop_for_status(GET(url = x, config = config, ...))
}

safe_GET <- safely(my_GET)

# If it can't connect to the bicing API will throw an error
test_bike <- safe_GET(test_url)

# If this is the time of the server, then it's a problem because
# it needs to be converted to Barcelona time. I think I had to
# convert the time to Barcelona time at some point.
current_time <- as.character(lubridate::now())

empty_df <-
  data.frame(
    id = NA,
    type = NA,
    latitude = NA,
    longitude = NA,
    streetName = NA,
    streetNumber = NA,
    altitude = NA,
    slots = NA,
    bikes = NA,
    nearbyStations = NA,
    status = NA,
    time = NA,
    error_msg = NA
  )

# If error in connecting to API return empty df with error message
if (!is.null(test_bike$error)) {
  semi_df <- empty_df
  semi_df$error_msg <- as.character(test_bike$error)
  semi_df$time <- current_time
  summary_bicing <- semi_df
} else {
  # continue with the usual scraping
  bicycle_url <- content(test_bike$result)$result$results[[1]]$url
  
  # turn that my_GET so that if there's an error the computation doesn't stop
  # but changes the result based on that error
  
  safe_request <- safe_GET(bicycle_url)
  
  # If there's an error, return an empty df with the error in the error column
  print("Error:", safe_request$error)
  
  # If error in requesting bicing data, returne empty df with error msg
  if (!is.null(safe_request$error)) {
    semi_df <- empty_df
    semi_df$error_msg <- as.character(test_bike$error)
    semi_df$time <- current_time
    summary_bicing <- semi_df
  } else {
    # scrape away
    bicing <- fromJSON(rawToChar(safe_request$result$content))$stations
    summary_bicing <- bicing
    
    print(paste0("Dim after subsetting station: ", dim(summary_bicing)))
    summary_bicing$time <- current_time
    summary_bicing$error_msg <- NA
    print(paste0("Dim after time and error vars: ", dim(summary_bicing)))
    
    summary_bicing
  }
}