Concise Recursive DynamoDB Scan - Large Table - Using ExclusiveStartKey and LastEvaluatedKey - NodeJS

gistfile1.txt

let params = {
  TableName:  'xxx',
  Limit: 50 // Configure based on needs
};
let aItems = [];
const recursiveScan = (params) => {
    return client.scan(params).promise().then((data) => {
        //  Simple Changes to input, optional
        let newItems = data.Items.map((item) => {
            return item;
        });
        aItems = aItems.concat(newItems);
        if(data.LastEvaluatedKey != null){
            params.ExclusiveStartKey = data.LastEvaluatedKey;
            //  Recursive call, as deep as we can loop !
            return recursiveScan(params);
        }
        return Promise.resolve(aItems);
    }).then((items) => {
        if(items != null && items.length != null)
            console.log("Final List : " + items.length);
    }).catch((error) => {
        console.log(error);
        console.log(JSON.stringify(error));
    });
};

how to process when exceed the rcu?

@deepanprabhu thanks for this gist; it's a great starting point! 🎉

Anyone finding this gist in the future the complete code to make this work is:

// Load the AWS SDK for JS
const AWS = require("aws-sdk");

// Set a region to interact with (make sure it's the same as the region of your table)
AWS.config.update({region: 'eu-west-2'});

// Set a table name that we can use later on
const tableName = "users"

// Create the Service interface for DynamoDB
const client = new AWS.DynamoDB({apiVersion: '2012-08-10'});

// Define the params we want to query Dynamo with
const params = {
  TableName:  tableName,
  Limit: 50 // Configure based on typical document size
};

let aItems = [];

const recursiveScan = (params) => {
    return client.scan(params).promise().then((data) => {
        //  Simple Changes to input, optional
        let newItems = data.Items.map((item) => {
            return item;
        });
        aItems = aItems.concat(newItems);
        if(data.LastEvaluatedKey != null){
            params.ExclusiveStartKey = data.LastEvaluatedKey;
            //  Recursive call, as deep as we can loop !
            return recursiveScan(params);
        }
      
        return Promise.resolve(aItems);
    }).then((items) => {
        if(items != null && items.length != null)
            console.log("Final List : " + items.length);
            return items;
    }).catch((error) => {
        console.log(error);
        console.log(JSON.stringify(error));
    });
};

// invoke the recursiveScan function and log the result:
( async function() {
  const items = await recursiveScan(params);
  console.log(items.length);
  console.log(JSON.stringify(items[0], null, 2));
})();

🌻

Thanks @nelsonic

You know it @deepanprabhu 😉 https://xkcd.com/979/