Created
August 13, 2014 02:43
-
-
Save DrkSephy/73e710af4ac0ce908603 to your computer and use it in GitHub Desktop.
Github API Pagination Algorithm for Node
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
PROBLEM: We do not know how many iterations we'll need to perform to access all of the data. | |
We want an algorithm to iterate over a page of data, and under some conditions | |
continue to run the requests function repeatedly until we reach a break condition. | |
Essentially, the runtime of the loop is dynamically changing and we need a way to | |
handle an indeterminate number of requests. | |
*/ | |
// Starting page counter | |
var pageCounter = 1; | |
// Given a base URL to start with... | |
var urls = [ GITHUB_RESOURCE_API + '/' + &page= ]; | |
request({ | |
// grab data for the first page of any resource | |
url: urls[0] + pageCounter, | |
headers: { 'user-agent': 'git-technetium' }, | |
json: true | |
}, function(error, response, body){ | |
if(!error && response.statusCode === 200){ | |
// Check the length of the body. If it is less than 30, then we know there is only one page of data. | |
if(body.length < 30){ | |
// ... handle the data as we normally would | |
} | |
else{ | |
// length of body is <= 30. This means there could potentially be another page of data. | |
// ... process data as we normally would | |
// Update pageCounter | |
pageCounter++; | |
// Push second page for a new request | |
urls.push(url + pageCounter); | |
// Now we want to keep calling the requests function recursively until we find a body length < 30 | |
} | |
} | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment