Utility that fetches from a paginated resource by repeatedly fetching & concatenating pages

fetch_paginated.js

// Fetch several rows from 1 or more pages from an API. Concatenate all
// the rows together.
function fetch_paginated(
  // The URL to initiate the query
  initial_url,

  // A fetch function that retrieves a single page. Intended to wrap a fetch()
  // API call (so that you can add your own headers, auth, response parsing,
  // etc)
  fetcher,

  // Given a response, get the URL of the next page, or null if this was the
  // last page.
  next_url,

  // Given a response, get all the rows in the page. All the rows from all the
  // pages will be concatenated for the final response from fetch_paginated.
  // If each page is its own row, this can simply be `response => [response]`
  get_rows,

  // Optional cancel signal, used to cancel this request. See
  // https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal
  // for details.
  cancel,
) {
  const fetch_page = (url, rows) =>
    fetcher(url, cancel).then((response) => {
      const page_rows = get_rows(response);
      const all_rows = rows.concat(page_rows);
      const next_page = next_url(response);

      return next_page == null ? all_rows : fetch_page(next_page, all_rows);
    });

  return fetch_page(initial_url, []);
}