Example AWS S3 Multipart Upload with aws-sdk for Node.js - Retries to upload failing parts

aws-multipartUpload.js

// Based on Glacier's example: http://docs.aws.amazon.com/AWSJavaScriptSDK/guide/examples.html#Amazon_Glacier__Multi-part_Upload
var fs = require('fs');
var AWS = require('aws-sdk');
AWS.config.loadFromPath('./aws-config.json');
var s3 = new AWS.S3();

// File
var fileName = '5.pdf';
var filePath = './' + fileName;
var fileKey = fileName;
var buffer = fs.readFileSync('./' + filePath);
// S3 Upload options
var bucket = 'loctest';

// Upload
var startTime = new Date();
var partNum = 0;
var partSize = 1024 * 1024 * 5; // Minimum 5MB per chunk (except the last part) http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
var numPartsLeft = Math.ceil(buffer.length / partSize);
var maxUploadTries = 3;
var multiPartParams = {
    Bucket: bucket,
    Key: fileKey,
    ContentType: 'application/pdf'
};
var multipartMap = {
    Parts: []
};

function completeMultipartUpload(s3, doneParams) {
  s3.completeMultipartUpload(doneParams, function(err, data) {
    if (err) {
      console.log("An error occurred while completing the multipart upload");
      console.log(err);
    } else {
      var delta = (new Date() - startTime) / 1000;
      console.log('Completed upload in', delta, 'seconds');
      console.log('Final upload data:', data);
    }
  });
}

function uploadPart(s3, multipart, partParams, tryNum) {
  var tryNum = tryNum || 1;
  s3.uploadPart(partParams, function(multiErr, mData) {
    if (multiErr){
      console.log('multiErr, upload part error:', multiErr);
      if (tryNum < maxUploadTries) {
        console.log('Retrying upload of part: #', partParams.PartNumber)
        uploadPart(s3, multipart, partParams, tryNum + 1);
      } else {
        console.log('Failed uploading part: #', partParams.PartNumber)
      }
      return;
    }
    multipartMap.Parts[this.request.params.PartNumber - 1] = {
      ETag: mData.ETag,
      PartNumber: Number(this.request.params.PartNumber)
    };
    console.log("Completed part", this.request.params.PartNumber);
    console.log('mData', mData);
    if (--numPartsLeft > 0) return; // complete only when all parts uploaded

    var doneParams = {
      Bucket: bucket,
      Key: fileKey,
      MultipartUpload: multipartMap,
      UploadId: multipart.UploadId
    };

    console.log("Completing upload...");
    completeMultipartUpload(s3, doneParams);
  });
}

// Multipart
console.log("Creating multipart upload for:", fileKey);
s3.createMultipartUpload(multiPartParams, function(mpErr, multipart){
  if (mpErr) { console.log('Error!', mpErr); return; }
  console.log("Got upload ID", multipart.UploadId);

  // Grab each partSize chunk and upload it as a part
  for (var rangeStart = 0; rangeStart < buffer.length; rangeStart += partSize) {
    partNum++;
    var end = Math.min(rangeStart + partSize, buffer.length),
        partParams = {
          Body: buffer.slice(rangeStart, end),
          Bucket: bucket,
          Key: fileKey,
          PartNumber: String(partNum),
          UploadId: multipart.UploadId
        };

    // Send a single part
    console.log('Uploading part: #', partParams.PartNumber, ', Range start:', rangeStart);
    uploadPart(s3, multipart, partParams);
  }
});

@yiweirong if still having that issue: you have to make sure that the s3.completeMultipartUpload function gets called after all the parts have been uploaded and that multipartMap.Parts is complete with information about all the parts.
i just had that issue and fixed it by doing that.

I should ask . Can't you call list parts , instead of manually saving the parts that have been uploaded .

Thanks for the code.

Thank you for the code.Very helpfull

    /**
     * initiate a multipart upload and get an upload ID that must include in upload part request.
     * Each part must be at least 5 MB in size, except the last part.
     */
    async multiPart(options) {
        const { data, bucket, key } = options;
        const multiPartParams = {
            Bucket: bucket,
            Key: key
        };
        const multipart = await this._client.createMultipartUpload(multiPartParams).promise();
        const multipartMap = { Parts: [] };
        let partNum = 0;
        for (const d of data) {
            partNum += 1;
            const partParams = { ...multiPartParams, Body: d, PartNumber: String(partNum), UploadId: multipart.UploadId };
            const result = await this._client.uploadPart(partParams).promise();
            multipartMap.Parts[partNum - 1] = { ETag: result.ETag, PartNumber: Number(partNum) };
        }
        const doneParams = { ...multiPartParams, MultipartUpload: multipartMap, UploadId: multipart.UploadId };
        const result = await this._client.completeMultipartUpload(doneParams).promise();
        return result;
    }

Why use it instead of .upload() function from aws? @nassiharel

https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property

@marceloavf is right. Using .upload() you get automatic multipart uploads with concurrency and retries.

const AWS = require('aws-sdk');
const mime = require('mime-types');
const endpoint = new AWS.Endpoint(S3_ENDPOINT);

const S3 = new AWS.S3({
	endpoint: endpoint,
	accessKeyId: S3_KEY,
	secretAccessKey: S3_SECRET,
	maxRetries: 10
});

const stream = fs.createReadStream(filePath);
const contentType = mime.lookup(filePath)

const params = {
	Bucket: BUCKET_NAME,
	Key: OBJECT_KEY,
	Body: stream,
	ACL: 'public-read',
	ContentType: contentType
};

const options = {
	partSize: 10 * 1024 * 1024,
        // how many concurrent uploads
	queueSize: 5
};

try {
	await S3.upload(params, options).promise();
	console.log('upload OK', filePath);
} catch (error) {
	console.log('upload ERROR', filePath, error);
}

Yes, you are right @marceloavf, @PierBover.
But this is a multi-part gist, so I just shared my part if someone still wants to use the multi-part API.
There is also an API for getting multi-parts.

@PierBover how would one check upload progress?

@PierBover figured it out heres the snippet

   try {
   	await s3.upload(params, options)
    .on('httpUploadProgress', function(evt) {

     console.log("Uploaded :: " + parseInt((evt.loaded * 100) / evt.total)+'%');
   })
    .promise();
   	console.log('upload OK' + oldpath);

   } catch (error) {

   	console.log('upload ERROR' + error);
   }

@PierBover figured it out heres the snippet

   try {
   	await s3.upload(params, options)
    .on('httpUploadProgress', function(evt) {

     console.log("Uploaded :: " + parseInt((evt.loaded * 100) / evt.total)+'%');
   })
    .promise();
   	console.log('upload OK' + oldpath);

   } catch (error) {

   	console.log('upload ERROR' + error);
   }

hi, the "httpUploadProgress" showing after uploaded complete to s3 on uploading its not firing the progress event, please advice.

@marceloavf is right. Using .upload() you get automatic multipart uploads with concurrency and retries.

const AWS = require('aws-sdk');
const mime = require('mime-types');
const endpoint = new AWS.Endpoint(S3_ENDPOINT);

const S3 = new AWS.S3({
	endpoint: endpoint,
	accessKeyId: S3_KEY,
	secretAccessKey: S3_SECRET,
	maxRetries: 10
});

const stream = fs.createReadStream(filePath);
const contentType = mime.lookup(filePath)

const params = {
	Bucket: BUCKET_NAME,
	Key: OBJECT_KEY,
	Body: stream,
	ACL: 'public-read',
	ContentType: contentType
};

const options = {
	partSize: 10 * 1024 * 1024,
        // how many concurrent uploads
	queueSize: 5
};

try {
	await S3.upload(params, options).promise();
	console.log('upload OK', filePath);
} catch (error) {
	console.log('upload ERROR', filePath, error);
}

@marceloavf, @PierBover. Do you have the code for the full async function for this please?

You can take a look here https://github.com/marceloavf/digitalocean-tools-vsts/blob/master/Tasks/DigitalOceanSpacesUpload/utils/Upload.ts

Thanks @marceloavf but this code is quite different from the code @PierBover posted and I am notusing Tyoescript. Do you have any code for the async function for this @PierBover?

@nickwild-999 just put the example into an async function. My example is already using await.

@marceloavf is right. Using .upload() you get automatic multipart uploads with concurrency and retries.

const AWS = require('aws-sdk');
const mime = require('mime-types');
const endpoint = new AWS.Endpoint(S3_ENDPOINT);

const S3 = new AWS.S3({
	endpoint: endpoint,
	accessKeyId: S3_KEY,
	secretAccessKey: S3_SECRET,
	maxRetries: 10
});

const stream = fs.createReadStream(filePath);
const contentType = mime.lookup(filePath)

const params = {
	Bucket: BUCKET_NAME,
	Key: OBJECT_KEY,
	Body: stream,
	ACL: 'public-read',
	ContentType: contentType
};

const options = {
	partSize: 10 * 1024 * 1024,
        // how many concurrent uploads
	queueSize: 5
};

try {
	await S3.upload(params, options).promise();
	console.log('upload OK', filePath);
} catch (error) {
	console.log('upload ERROR', filePath, error);
}

Try in Node 14?

Has anybody been able to set Metadata fields with createMultipartUpload?

This might help.
const params = {
ACL: "public-read",
Key: "sample-videofile",
ContentType: video/mp4,
Body: file,
ContentLength: file.size,
Metadata: {
user_email: "[email protected]",
user_id: "300",
},
};

Upload with progress bar...this is a tested code.

S3.upload(params, options, function (err, data) {
          if (err) {
            reject("error");
          }
          alert("Successfully Uploaded!");
        }).on("httpUploadProgress", (progress) => {
          let uploaded = parseInt((progress.loaded * 100) / progress.total);
          this.setState({
            progress: uploaded,
            uploadText: "Uploading...",
            uploading: true,
          });
        });

data

How to call the above function for multiple large files

Example:

FileList - [file1, file2]

let PromiseArray = []

for(file in FileList){
PromiseArray.push(multiPart(file))
}
Promise.all(PromiseArray)
.then(result => {
//succese
})

when I can parallel the multiPart function its mismatch file 1 and file 2 data and overwrite one of the file for both

Any Solution ?

    /**
     * initiate a multipart upload and get an upload ID that must include in upload part request.
     * Each part must be at least 5 MB in size, except the last part.
     */
    async multiPart(options) {
        const { data, bucket, key } = options;
        const multiPartParams = {
            Bucket: bucket,
            Key: key
        };
        const multipart = await this._client.createMultipartUpload(multiPartParams).promise();
        const multipartMap = { Parts: [] };
        let partNum = 0;
        for (const d of data) {
            partNum += 1;
            const partParams = { ...multiPartParams, Body: d, PartNumber: String(partNum), UploadId: multipart.UploadId };
            const result = await this._client.uploadPart(partParams).promise();
            multipartMap.Parts[partNum - 1] = { ETag: result.ETag, PartNumber: Number(partNum) };
        }
        const doneParams = { ...multiPartParams, MultipartUpload: multipartMap, UploadId: multipart.UploadId };
        const result = await this._client.completeMultipartUpload(doneParams).promise();
        return result;
    }

this._client.createMultipartUpload is available in which npm package ?
Please Reply
Does this code work ??

Just one question. Does this occupy the nodejs server's disk space during the upload process?

thanks for the input!

I created a similar script based on the async package including retries that might be a bit easier to use & understand for some ppl!

https://gist.github.com/magegu/ea94cca4a40a764af487

I am looking into your script, and wondering if there is a way i can resume broken uploads? For eg if my frontend app has uploaded PartNumber:2 and then encountered a broken internet connection, is it possible to resume the upload from where it got last uploaded i.e. partNumber2 instead of uploading from 0?

    /**
     * initiate a multipart upload and get an upload ID that must include in upload part request.
     * Each part must be at least 5 MB in size, except the last part.
     */
    async multiPart(options) {
        const { data, bucket, key } = options;
        const multiPartParams = {
            Bucket: bucket,
            Key: key
        };
        const multipart = await this._client.createMultipartUpload(multiPartParams).promise();
        const multipartMap = { Parts: [] };
        let partNum = 0;
        for (const d of data) {
            partNum += 1;
            const partParams = { ...multiPartParams, Body: d, PartNumber: String(partNum), UploadId: multipart.UploadId };
            const result = await this._client.uploadPart(partParams).promise();
            multipartMap.Parts[partNum - 1] = { ETag: result.ETag, PartNumber: Number(partNum) };
        }
        const doneParams = { ...multiPartParams, MultipartUpload: multipartMap, UploadId: multipart.UploadId };
        const result = await this._client.completeMultipartUpload(doneParams).promise();
        return result;
    }

Hi! Your solution looks very good, but i don't understand what kind of data i should use? I mean your data looks like it is an iterable object

@jotta008 Thank you !
Will this work with file size of 600 mb?