A AWS Lambda function to in-plae Gzip compress all JSON files in AWS S3 bucket

gzip-json-files-in-aws-s3.ts

/**
This is an AWS Lambda function which, when run, Gzip compressess all JSON files in AWS S3 bucket.

Define BUCKET_NAME environment variable to specify the bucket.

A new file is written based on the object name. For example "foo.json" is compressed to "foo.json.gz" and the old object is removed.
*/

import { S3, ListObjectsV2Output, ListObjectsV2CommandOutput } from '@aws-sdk/client-s3';
import * as zlib from 'zlib';
import { Readable } from 'stream';

const bucket = process.env.BUCKET_NAME ?? '';
const s3 = new S3({});

const processContents = async (files: ListObjectsV2Output) => {
  if (!files.Contents) {
    throw new Error('Invalid files.Contents');
  }

  const fileBatch = files.Contents.map(async (currentObject) => {
    if (currentObject.Key?.endsWith('.json')) {
        console.log(`Processing ${currentObject.Key}`);

        const s3Result = await s3.getObject({
          Bucket: bucket,
          Key: currentObject.Key,
        });

        if (!s3Result.Body) {
          throw new Error(`Empty Body ${currentObject.Key}`);
        }

        const stream = s3Result.Body as Readable;

        const json = await new Promise<Buffer>((resolve, reject) => {
          const chunks: Buffer[] = []
          stream.on('data', (chunk) => chunks.push(chunk))
          stream.once('end', () => resolve(Buffer.concat(chunks)))
          stream.once('error', reject)
        });

        await s3.putObject({
          Bucket: bucket,
          Key: currentObject.Key + '.gz',
          Body: zlib.gzipSync(json),
        });
        console.log(`Wrote ${currentObject.Key}.gz`);

        await s3.deleteObject({
          Bucket: bucket,
          Key: currentObject.Key,
        });
        console.log(`Deleted ${currentObject.Key}`);
    }
  });
  await Promise.all(fileBatch);
}

export const handler = async (event: any) => {
  console.log(JSON.stringify(event, undefined, 2));
  try {
    var continuationToken: string | undefined = undefined;
    do {
      const files: ListObjectsV2CommandOutput = await s3.listObjectsV2({
        Bucket: bucket,
        ContinuationToken: continuationToken,
      });
      continuationToken = files.NextContinuationToken;
      await processContents(files);
    } while(continuationToken !== undefined);

    return {
      statusCode: 200,
    };
  } catch (e) {
    console.error(e);
    return {
      statusCode: 500,
      body: JSON.stringify(e)
    };
  }
};

infra-aws-cdk-lambda.ts

    const convert = new lambda_nodejs.NodejsFunction(this, 'Convert', {
      runtime: lambda.Runtime.NODEJS_16_X,
      bundling: { minify: false, },
      memorySize: 1024,
      timeout: cdk.Duration.minutes(15),
      architecture: lambda.Architecture.ARM_64,
      environment: {
        BUCKET_NAME: bucket.bucketName,
      },
    });
    bucket.grantReadWrite(convert);
    bucket.grantDelete(convert);