scrape-zips.md

library(httr)
library(readr)
library(dplyr)
library(tidyr)

HEADERS = c(
  accept = "*/*",
  `accept-language` = "en-US,en;q=0.9",
  `content-type` = "application/json; charset=utf-8",
  origin = "https://www.elrexfio.com",
  priority = "u=1, i",
  `sec-ch-ua` = '"Chromium";v="140", "Not=A?Brand";v="24", "Google Chrome";v="140"',
  `sec-ch-ua-mobile` = "?0",
  `sec-ch-ua-platform` = '"macOS"',
  `sec-fetch-dest` = "empty",
  `sec-fetch-mode` = "cors",
  `sec-fetch-site` = "cross-site",
  `user-agent` = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
  `x-config-token` = "prod__rep_locator_elrexfio"
)
DATA_DIR <- '~/downloads/zip_code'
dir.create(DATA_DIR, showWarnings = FALSE)
OVERWRITE <- FALSE
POST_DATA_TEMPLATE <- '{"csrfToken":"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjb25maWdUb2tlbiI6InByb2RfX3JlcF9sb2NhdG9yX2VscmV4ZmlvIiwiaWQiOiIyMzQ1ZDlmMDZjMTZjNmQ3Y2I1NmIxM2NhNDQ3M2U2ZCIsImV4cCI6MTc1OTk2ODY1MCwiaXNzIjoiREVUIiwiYXVkIjoiaHR0cHM6Ly93d3cuZWxyZXhmaW8uY29tIn0.4gwwd5IYfdx6BvvKvq4kQ90ti8doOrAQyOOHLKOEbsk","request_id":"45365806","zipCode":"<<zip_code>>", "radius":50}'

get_zip_code_data <- function(zip_code) {
  post_data <- glue::glue(POST_DATA_TEMPLATE, .open = '<<', .close = '>>')

  resp <- httr::POST(
    "https://ms-forms-service-production.digitalpfizer.com/api/v2/forms",
    httr::add_headers(.headers = headers),
    body = post_data
  )

  cont <- httr::content(resp)

  data_start <- cont[['data']][[1]]
  if (length(data_start) == 0) {
    return(tibble::tibble())
  }

  participants <- data_start[['details']][['participants']]
  if (length(participants) == 0) {
    return(tibble::tibble())
  }

  tibble::enframe(
    participants
  ) |>
    dplyr::select(value) |>
    tidyr::unnest_wider(value)
}

ZIP_CODES <- c(
  78660,
  78723
)

zip_code_results <- purrr::map_dfr(
  ZIP_CODES,
  \(.x) {

    path <- file.path(DATA_DIR, paste0(.x, '.rds'))
    if (file.exists(path) & !OVERWRITE) {
      return(readr::read_rds(path))
    }

    Sys.sleep(runif(1, min = 0.5, max = 2))
    res <- get_zip_code_data(.x)
    readr::write_rds(res, path)
    res
  }
)

1. Copy as CURL on request.

(See "Response" tab to know it's the right request.)

2. Paste in https://curlconverter.com/r/ to get base code.
3. ???
4. Profit