Skip to content

Instantly share code, notes, and snippets.

@Max-Makhrov
Last active May 11, 2021 00:23
Show Gist options
  • Save Max-Makhrov/f05d5de281551efb2117836383f81e1e to your computer and use it in GitHub Desktop.
Save Max-Makhrov/f05d5de281551efb2117836383f81e1e to your computer and use it in GitHub Desktop.
function test_parse() {
var file = SpreadsheetApp.getActive();
var sheet = file.getSheetByName('Sheet1'); // change
var sets = {
rangeParse: sheet.getRange('L2:L16429'), // change
rangeResult: sheet.getRange("M2"), // change
func_parse: parseOkkoFilmToGetHorizontalPoster_ // change / create your function
};
parseUrls_(sets);
}
// Stage [ 0 ]. Define function for parsing conent text
function parseOkkoFilmToGetHorizontalPoster_(text) {
var re = new RegExp("v2\/([^?]*)\?");
var found = text.match(re);
if (!found) { return '-'; }
return found[0];
}
// Stage [ 1 ]. Get All requests to complete as 2d array (column)
// sets:
// {
// rangeParse,
// rangeResult
// func_parse
// }
function parseUrls_(sets) {
var limit = 50; // parse by 50 urls
var res = []; // 2d-array
//
var column = sets.rangeParse.getValues();
var rto = sets.rangeResult.offset(0, 0, 1, 1);
//
var t = new Date();
//
var requests = [], url, datares = [];
for (var i = 0; i < column.length; i++) {
url = column[i][0];
if (!url || url === '#N/A') { url = 'https://max-makhrov.github.io/'; }
var request = {
url: url,
muteHttpExceptions: true
}
requests.push(request);
if ((i % 50) === 0 && i !== 0) {
datares = parseRequestsTo2DResult_(requests, sets.func_parse);
rto.offset(i - datares.length + 1, 0, datares.length).setValues(datares);
requests = [];
}
}
if (requests.length > 0) {
datares = parseRequestsTo2DResult_(requests, sets.func_parse);
rto.offset(i - datares.length, 0, datares.length).setValues(datares);
}
console.log(new Date() - t)
}
// Stage [ 2 ]. Parse requests
function parseRequestsTo2DResult_(requests, func) {
var responses = UrlFetchApp.fetchAll(requests);
var res0 = responses.map(function (e) { return e.getContentText() });
var res = []; // 2d array
for (var i = 0; i < res0.length; i++) {
res.push([func(res0[i])]);
}
return res;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment