Query JSON with S3 Select in Node.js

example-query.js

// See tth.im/s3json for a full explanation of this code

const AWS = require('aws-sdk');
const S3 = new AWS.S3();

exports.handler = async (_, context) => {
  try {
    const query = 'SELECT * FROM s3object[*].results[*] r;';
    
    // test query is against data from https://swapi.co/api/planets/?page=2
    const bucket = 'test-bucket-for-s3-select-tutorial';
    const key = 'planets.json';

    const params = {
      Bucket: bucket,
      Key: key,
      ExpressionType: 'SQL',
      Expression: query,
      InputSerialization: {
        JSON: {
          Type: 'DOCUMENT',
        }
      },
      OutputSerialization: {
        JSON: {
          RecordDelimiter: ','
        }
      }
    }

    const data = await getDataUsingS3Select(params);

    context.succeed(data);
  } catch (error) {
    context.fail(error);
  }
};

const getDataUsingS3Select = async (params) => {
  return new Promise((resolve, reject) => {
    S3.selectObjectContent(params, (err, data) => {
      if (err) { reject(err); }

      if (!data) {
        reject('Empty data object');
      }

      // This will be an array of bytes of data, to be converted
      // to a buffer
      const records = []

      // This is a stream of events
      data.Payload.on('data', (event) => {
        // There are multiple events in the eventStream, but all we 
        // care about are Records events. If the event is a Records 
        // event, there is data inside it
        if (event.Records) {
          records.push(event.Records.Payload);
        }
      })
      .on('error', (err) => {
        reject(err);
      })
      .on('end', () => {
        // Convert the array of bytes into a buffer, and then
        // convert that to a string
        let planetString = Buffer.concat(records).toString('utf8');

        // remove any trailing commas
        planetString = planetString.replace(/\,$/, '');

        // Add into JSON 'array'
        planetString = `[${planetString}]`;

        try {
          const planetData = JSON.parse(planetString);
          resolve(planetData);
        } catch (e) {
          reject(new Error(`Unable to convert S3 data to JSON object. S3 Select Query: ${params.Expression}`));
        }
      });
    });
  })
}