Skip to content

Instantly share code, notes, and snippets.

@chipoglesby
Last active December 6, 2018 14:42
Show Gist options
  • Select an option

  • Save chipoglesby/a9c821eec94fb72ba40a to your computer and use it in GitHub Desktop.

Select an option

Save chipoglesby/a9c821eec94fb72ba40a to your computer and use it in GitHub Desktop.
An example script of how to load data via an API (in this case, CallRail) into Google BigQuery as a newline JSON file
function bigQueryRun() {
projectId = "xxx"; // Replace xxx with your project id
datasetId = "xxx"; // Replace xxx with your dataset id
tableId = "xxx"; // Replace xxx with your
yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 29);
var yesterday = Utilities.formatDate(yesterday, 'UTC', 'yyyy-MM-dd');
options = {"headers": {"authorization": "Token token=\"xxx\""}}; // Replace xxx with your API token
response = UrlFetchApp.fetch('https://api.callrail.com/v1/calls.json?start_date=2015-01-01&end_date='+yesterday+'&per_page=250',options);
pages = JSON.parse(response.getContentText());
total_pages = pages.total_pages;
data = [];
for (var i = 1; i < total_pages;i++) {
response = UrlFetchApp.fetch('https://api.callrail.com/v1/calls.json?start_date=2015-01-01&end_date='+yesterday+'&per_page=250&page='+[i],options);
dataAll = JSON.parse(response.getContentText());
for (j in dataAll.calls) {
calls = dataAll.calls[j];
callPush = JSON.stringify({'caller_state':calls.caller_state,
'caller_zip':calls.caller_zip,
'caller_city':calls.caller_city,
'answered':calls.answered,
'caller_name':calls.caller_name,
'keywords':calls.keywords,
'utm_source':calls.utm_source,
'utm_medium':calls.utm_medium,
'utm_term':calls.utm_term,
'utm_content':calls.utm_content,
'utm_campaign':calls.campaign,
'gclid':calls.gclid,
'first_call':calls.first_call,
'customer_phone_number':calls.customer_phone_number,
'tracking_number':calls.tracking_number,
'formated_duration':calls.duration,
'created_at':calls.created_at,
'source_name':calls.source_name,
'source_type':calls.source_type,
'medium':calls.medium,
'ga':calls.ga,
'device_type':calls.device_type});
data.push(callPush);
}
}
var data = data.join("\n");
blobData = Utilities.newBlob(data, "application/octet-stream");
job = {
configuration: {
load: {
destinationTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
},
sourceFormat: "NEWLINE_DELIMITED_JSON",
}
}
}
job = BigQuery.Jobs.insert(job, projectId, blobData);
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment