Use a JS generator to create an iterator for paginated API responses

iterator.js

'use strict'
/** 
 * Objeect Oriented implementation of an iterable to handle paginated requests
 * 
 * @param {string} requestMethod   HTTP method for the request
 * @param {string} firstUrl        Endpoint for the request
 * 
 */
class PaginatedList {
  constructor(requestMethod, firstUrl) {
    this._firstUrl = firstUrl;
    this._requestMethod = requestMethod;
    
    // An array to be populated by the generator
    this._elements = [];

    this._nextUrl = firstUrl;
  }

  // This class is an iterable, so we need our own iterator, which is
  // a generator function.
  // https://www.javascripttutorial.net/es6/javascript-iterator/
  async *[Symbol.asyncIterator]() {
    for(let element in this._elements) {
      yield element
    }
    while(this.hasNext()) {
      let newElements = await this.grow();
      for await (let el of newElements) {
        yield el;
      }
    }
  }

  hasNext() {
    return this._nextUrl != null;
  }

  // Get more elements
  async grow() {
    let newElements = await this.getNextPage()
    this._elements += newElements;
    return newElements
  }

  async getNextPage() {
    let content = []
    const options = {
      "method": this._requestMethod,
      "headers": { "Content-Type": "application/json" }
    }

    let request = await fetch(this._nextUrl, options)
    let response = await request.json()

    // Get all links from the headers and check for "next" to continue
    let nextLink = this.parseLinkHeader(request.headers.get("Link"))
    if(nextLink["next"]) {
      this._nextUrl = nextLink["next"]
    } else {
      this._nextUrl = null;
    }

    for(let element of response) {
      if(element != null) {
        content.push(element)
      }
    }

    return content
  }

  // Turn a string of links into an object
  parseLinkHeader(header) {
    if (header.length === 0) {
        throw new Error("input must not be of zero length");
    }

    var parts = header.split(',');
    var links = {};

    for(var i=0; i<parts.length; i++) {
        var section = parts[i].split(';');
        if (section.length !== 2) {
            throw new Error("section could not be split on ';'");
        }
        var url = section[0].replace(/<(.*)>/, '$1').trim();
        var name = section[1].replace(/rel="(.*)"/, '$1').trim();
        links[name] = url;
    }
    return links;
  }
}

can you write an example of how to implement this with github api for repositories ?? https://api.github.com?page=1

I threw this up fast yesterday. I made a couple bug fixes that now allow you to get results asynchronously. More here on async generators and iterators.

To use it, you can do something like this:

let request = new PaginatedList('GET', 'https://api.github.com/orgs/octokit/repos')

(async () => {
    for await(let item of request) {
        console.log(item.name)
    }
})()

This is just a rough example, so there's definitely some cleanup to do, but it gets the job done most times. If you want to see each URL called, you can console.log(this._nextUrl) on line 52.