Last active
May 11, 2021 00:23
-
-
Save Max-Makhrov/f05d5de281551efb2117836383f81e1e to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
function test_parse() { | |
var file = SpreadsheetApp.getActive(); | |
var sheet = file.getSheetByName('Sheet1'); // change | |
var sets = { | |
rangeParse: sheet.getRange('L2:L16429'), // change | |
rangeResult: sheet.getRange("M2"), // change | |
func_parse: parseOkkoFilmToGetHorizontalPoster_ // change / create your function | |
}; | |
parseUrls_(sets); | |
} | |
// Stage [ 0 ]. Define function for parsing conent text | |
function parseOkkoFilmToGetHorizontalPoster_(text) { | |
var re = new RegExp("v2\/([^?]*)\?"); | |
var found = text.match(re); | |
if (!found) { return '-'; } | |
return found[0]; | |
} | |
// Stage [ 1 ]. Get All requests to complete as 2d array (column) | |
// sets: | |
// { | |
// rangeParse, | |
// rangeResult | |
// func_parse | |
// } | |
function parseUrls_(sets) { | |
var limit = 50; // parse by 50 urls | |
var res = []; // 2d-array | |
// | |
var column = sets.rangeParse.getValues(); | |
var rto = sets.rangeResult.offset(0, 0, 1, 1); | |
// | |
var t = new Date(); | |
// | |
var requests = [], url, datares = []; | |
for (var i = 0; i < column.length; i++) { | |
url = column[i][0]; | |
if (!url || url === '#N/A') { url = 'https://max-makhrov.github.io/'; } | |
var request = { | |
url: url, | |
muteHttpExceptions: true | |
} | |
requests.push(request); | |
if ((i % 50) === 0 && i !== 0) { | |
datares = parseRequestsTo2DResult_(requests, sets.func_parse); | |
rto.offset(i - datares.length + 1, 0, datares.length).setValues(datares); | |
requests = []; | |
} | |
} | |
if (requests.length > 0) { | |
datares = parseRequestsTo2DResult_(requests, sets.func_parse); | |
rto.offset(i - datares.length, 0, datares.length).setValues(datares); | |
} | |
console.log(new Date() - t) | |
} | |
// Stage [ 2 ]. Parse requests | |
function parseRequestsTo2DResult_(requests, func) { | |
var responses = UrlFetchApp.fetchAll(requests); | |
var res0 = responses.map(function (e) { return e.getContentText() }); | |
var res = []; // 2d array | |
for (var i = 0; i < res0.length; i++) { | |
res.push([func(res0[i])]); | |
} | |
return res; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment