Gets Items from a Large Document Library/List (over 5000 items) using a Paged RenderListDataAsStream

Get Paged Items from a Large List using RenderListDataAsStream

import { sp } from "@pnp/sp/presets/all";

(async () => {
  console.clear();

  const doPagedQuery = (folder: string, rowLimit: number, scope: "Files"|"Folders", pageToken?: string) => {
    const list = sp.web.getList('/sites/EDMS_InformationGovernance/Shared%20Documents');
    return list.renderListDataAsStream({
      ViewXml: `
        <View Scope="RecursiveAll">
          <RowLimit Paged="TRUE">${rowLimit}</RowLimit>
          <Query>
            <OrderBy Override="TRUE">
              <FieldRef Ascending="TRUE" Name="ID"/>
            </OrderBy>
            <Where>
              <And>
                <Or>
                  <Or>
                    <Eq>
                      <FieldRef Name="Edmsdisposition"/>
                      <Value Type="Text">Open</Value>
                    </Eq>
                    <Eq>
                      <FieldRef Name="Edmsdisposition"/>
                      <Value Type="Text">n/a</Value>
                    </Eq>
                  </Or>
                  <IsNull>
                    <FieldRef Name="Edmsdisposition"/>
                  </IsNull>
                </Or>
                <Eq>
                  <FieldRef Name="FSObjType"/>
                  <Value Type="Counter">${scope === "Files" ? 0 : 1}</Value>
                </Eq>
              </And>
            </Where>
          </Query>
          <ViewFields>
            <FieldRef Name="Title"/>
            <FieldRef Name="FileRef"/>
            <FieldRef Name="FSObjType"/>
            <FieldRef Name=\"FileDirRef\"/>
            <FieldRef Name="FileLeafRef"/>
            <FieldRef Name="UniqueId"/>
            <FieldRef Name="ContentTypeId"/>
            <FieldRef Name="Fileplanmarker"/>
            <FieldRef Name="Edmsdisposition"/>
            <FieldRef Name="Edmsdateclosed"/>
            <FieldRef Name="_IsRecord"/>
            <FieldRef Name="Author"/>
            <FieldRef Name="Modified"/>
            <FieldRef Name="Created"/>
          </ViewFields>
        </View>
      `,
      FolderServerRelativeUrl: `${folder}`,
      Paging: `${pageToken}`
    });
  };

  const getPagedItems = async (folder: string, scope: "Files"|"Folders") => {
    const pageSize = 100;
    let nextPageToken = "";
    let pageNumber = 0;
    let itemCount = 0;
    while (nextPageToken !== null) {
      console.log(`${scope}) PAGENUMBER: ${pageNumber}`);

      const pagedResponse = await doPagedQuery(folder, pageSize, scope, nextPageToken);
      if (pagedResponse.Row && pagedResponse.Row.length) {
        (pagedResponse.Row as any[]).forEach((item) => {
          console.log(`[${item.Edmsdisposition}],[${item.Edmsdateclosed}],[${item.Fileplanmarker.Label}],[${item.FSObjType}],[${item.ContentType}]\n--> ${item.FileDirRef}\n--> ${item.FileRef}`);
        });
        itemCount += pagedResponse.Row.length;
      } else {
        console.info("WEIRD RESPONSE: -->", pagedResponse);
      }

      nextPageToken = pagedResponse.NextHref && pagedResponse.NextHref.length ? pagedResponse.NextHref.split('?')[1] : null;
      pageNumber++;
    }
    console.log(`${scope}) TOTALITEMS: ${itemCount}, TOTALPAGES: ${pageNumber-1}`);
  };

  let folder = "/sites/EDMS_InformationGovernance/Shared Documents/Records Management/Committee Reports/Folder 00100";
  await getPagedItems(folder, "Files");
  await getPagedItems(folder, "Folders");

})().catch(console.error)