mutipart upload for aws s3 with nodejs based on the async lib including retries for part uploads

multipart.js

/*
by Martin Güther @magegu

just call it: 

uploadFile(absoluteFilePath, callback);
*/

var path = require('path');
var async = require('async');
var fs = require('fs');
var AWS = require('aws-sdk'); 

AWS.config.loadFromPath('./aws.json');

var s3 = new AWS.S3(); 
var bucketName = "YOUR BUCKET NAME";

function uploadMultipart(absoluteFilePath, fileName, uploadCb) {
  s3.createMultipartUpload({ Bucket: bucketName, Key: fileName }, (mpErr, multipart) => {
    if(!mpErr){
      //console.log("multipart created", multipart.UploadId);
      fs.readFile(absoluteFilePath, (err, fileData) => {

        var partSize = 1024 * 1024 * 5;
        var parts = Math.ceil(fileData.length / partSize);

        async.timesSeries(parts, (partNum, next) => {

          var rangeStart = partNum*partSize;
          var end = Math.min(rangeStart + partSize, fileData.length);

          console.log("uploading ", fileName, " % ", (partNum/parts).toFixed(2));

          partNum++;  
          async.retry((retryCb) => {
            s3.uploadPart({
              Body: fileData.slice(rangeStart, end),
              Bucket: bucketName,
              Key: fileName,
              PartNumber: partNum,
              UploadId: multipart.UploadId
            }, (err, mData) => {
              retryCb(err, mData);
            });
          }, (err, data)  => {
            //console.log(data);
            next(err, {ETag: data.ETag, PartNumber: partNum});
          });

        }, (err, dataPacks) => {
          s3.completeMultipartUpload({
            Bucket: bucketName,
            Key: fileName,
            MultipartUpload: {
              Parts: dataPacks
            },
            UploadId: multipart.UploadId
          }, uploadCb);
        });
      });
    }else{
      uploadCb(mpErr);
    }
  });
}

function uploadFile(absoluteFilePath, uploadCb) {
  var fileName = path.basename(absoluteFilePath);
  var stats = fs.statSync(absoluteFilePath)
  var fileSizeInBytes = stats["size"]

  if(fileSizeInBytes < (1024*1024*5)) {
    async.retry((retryCb) => {
      fs.readFile(absoluteFilePath, (err, fileData) => {
        s3.putObject({
          Bucket: bucketName, 
          Key: fileName, 
          Body: fileData
        }, retryCb);        
      });
    }, uploadCb);
  }else{
    uploadMultipart(absoluteFilePath, fileName, uploadCb)
  }
}

can we used read stream instead readfile?

Thanks for code.

RangeError [ERR_FS_FILE_TOO_LARGE]: File size (10485760000) is greater than possible Buffer: 2147483647 bytes

@shahrukhdev

I think that's an fs error and has nothing to do with s3.

Check this answer on stackoverflow, it might fix your issue.
https://stackoverflow.com/a/63553430/11084093

Thanks a lot @magegu , that was really helpful.
But you need to update the code to change (1024_1024_5)=5MB to (1024_1024_1024*5)=5GB

Why is that?

I am sorry that I am pretty new in node js, can anyone post a short demo of how to use this code for multipart upload a huge size file (like 20 Gb) from S3 bucket ( assume has no permission issue), thank you,.

I think you’re better off doing a Google search there. StackOverflow has a lot of solutions, and it’s better than waiting for a benefactor to do that snippet for you Christophe N. El  Khoury Software Engineer, Technical Consultant Lebanon Alternative ***@***.*** LinkedIn | GitHub | StackOverflow Please consider the environment before printing this email.

…

On 12 Jul 2021, 11:42 PM +0300, Youth Inno Lab ***@***.***>, wrote: @YouthInnoLab commented on this gist. I am sorry that I am pretty new in node js, can anyone post a short demo of how to use this code for multipart upload a huge size file (like 20 Gb) from S3 bucket ( assume has no permission issue), thank you,. — You are receiving this because you commented. Reply to this email directly, view it on GitHub, or unsubscribe.

Just one question. Does this occupy the nodejs server's disk space during the upload process?

Extremely helpful thanks. would it make sense as an NPM package?

@soubhikchatterjee I think it would, but you could swap memfs for fs to avoid that I think

https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property

Does the SDK handle this for us now or no?

This works fine in my end. Thanks a lot!