Multipart upload to S3

multipart_s3_upload.js

import fs from 'fs';
import { Buffer } from 'node:buffer';
import {
  S3Client,
  CreateMultipartUploadCommand,
  UploadPartCommand,
  CompleteMultipartUploadCommand
} from '@aws-sdk/client-s3';

// 100 MB chunk/part size
const CHUNK_SIZE = 1024 * 1024 * 100;

// Max retries when uploading parts
const MAX_RETRIES = 3;

const multipartS3Uploader = async (filePath, options) => {
  const { region, contentType, key, bucket } = options;

  // Get file size
  const fileSize = fs.statSync(filePath).size;

  // Calculate total parts
  const totalParts = Math.ceil(fileSize / CHUNK_SIZE);

  // Initialize the S3 client instance
  const S3 = new S3Client({ region });
  const uploadParams = { Bucket: bucket, Key: key, ContentType: contentType };

  let PartNumber = 1;
  const uploadPartResults = [];

  // Send multipart upload request to S3, this returns a UploadId for use when uploading individual parts
  // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/classes/s3.html#createmultipartupload
  let { UploadId } = await S3.send(new CreateMultipartUploadCommand(uploadParams));

  console.log(`Initiate multipart upload, uploadId: ${UploadId}, totalParts: ${totalParts}, fileSize: ${fileSize}`);

  // Read file parts and upload parts to s3, this promise resolves when all parts are uploaded successfully
  await new Promise(resolve => {
    fs.open(filePath, 'r', async (err, fileDescriptor) => {
      if (err) throw err;

      // Read and upload file parts until end of file
      while (true) {

        // Read next file chunk
        const { buffer, bytesRead } = await readNextPart(fileDescriptor);

        // When end-of-file is reached bytesRead is zero
        if (bytesRead === 0) {
          // Done reading file, close the file, resolve the promise and return
          fs.close(fileDescriptor, (err) => { if (err) throw err; });
          return resolve();
        }

        // Get data chunk/part
        const data = bytesRead < CHUNK_SIZE ? buffer.slice(0, bytesRead) : buffer;

        // Upload data chunk to S3
        const response = await uploadPart(S3,
          { data, bucket, key, PartNumber, UploadId }
        );

        console.log(`Uploaded part ${PartNumber} of ${totalParts}`);
        uploadPartResults.push({ PartNumber, ETag: response.ETag });
        PartNumber++;
      }
    });
  });

  console.log(`Finish uploading all parts for multipart uploadId: ${UploadId}`);

  // Completes a multipart upload by assembling previously uploaded parts.
  // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/classes/s3.html#completemultipartupload
  let completeUploadResponse = await S3.send(new CompleteMultipartUploadCommand({
    Bucket: bucket,
    Key: key,
    MultipartUpload: { Parts: uploadPartResults },
    UploadId: UploadId
  }));

  console.log('Successfully completed multipart upload');

  return completeUploadResponse;
};

const readNextPart = async (fileDescriptor) => await new Promise((resolve, reject) => {
  // Allocate an empty buffer to save data chunk that is read
  const buffer = Buffer.alloc(CHUNK_SIZE);

  fs.read(
    fileDescriptor,
    buffer,                             // Buffer where data will be written
    0,                                  // Start Offset on buffer while writing data
    CHUNK_SIZE,                         // Length of bytes to read
    null,                               // Position in file(PartNumber * CHUNK_SIZE); if position is null data is read from the current file position, and the position is updated
    (err, bytesRead) => {               // Callback function
      if (err) return reject(err);
      resolve({ bytesRead, buffer });
    });
});

// Upload a given part with retries
const uploadPart = async (S3, options, retry = 1) => {
  const { data, bucket, key, PartNumber, UploadId } = options;
  let response;
  try {
    // Upload part to S3
    // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/classes/s3.html#uploadpart
    response = await S3.send(
      new UploadPartCommand({
        Body: data,
        Bucket: bucket,
        Key: key,
        PartNumber,
        UploadId
      })
    );
  } catch {
    console.log(`ATTEMPT-#${retry} Failed to upload part ${PartNumber} due to ${JSON.stringify(response)}`);

    if (retry >= MAX_RETRIES)
      throw (response);
    else
      return uploadPart(S3, options, retry + 1);
  }

  return response;
};

export default multipartS3Uploader;

// Example:

// await multipartS3Uploader(<file path>,
//   {
//     region: 'us-east-1',
//     bucket: 'my-s3-bucket',
//     key: <file path in bucket>
//   }
// );

// Try running the script like:
// AWS_ACCESS_KEY_ID=xxxx AWS_SECRET_ACCESS_KEY=xxxx node index.js

Thanks for creating this bucket, new aws s3 SDK has shamefully small number of examples, this one has proven to be most useful.