Last active
July 11, 2023 20:12
-
-
Save magegu/ea94cca4a40a764af487 to your computer and use it in GitHub Desktop.
mutipart upload for aws s3 with nodejs based on the async lib including retries for part uploads
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
by Martin Güther @magegu | |
just call it: | |
uploadFile(absoluteFilePath, callback); | |
*/ | |
var path = require('path'); | |
var async = require('async'); | |
var fs = require('fs'); | |
var AWS = require('aws-sdk'); | |
AWS.config.loadFromPath('./aws.json'); | |
var s3 = new AWS.S3(); | |
var bucketName = "YOUR BUCKET NAME"; | |
function uploadMultipart(absoluteFilePath, fileName, uploadCb) { | |
s3.createMultipartUpload({ Bucket: bucketName, Key: fileName }, (mpErr, multipart) => { | |
if(!mpErr){ | |
//console.log("multipart created", multipart.UploadId); | |
fs.readFile(absoluteFilePath, (err, fileData) => { | |
var partSize = 1024 * 1024 * 5; | |
var parts = Math.ceil(fileData.length / partSize); | |
async.timesSeries(parts, (partNum, next) => { | |
var rangeStart = partNum*partSize; | |
var end = Math.min(rangeStart + partSize, fileData.length); | |
console.log("uploading ", fileName, " % ", (partNum/parts).toFixed(2)); | |
partNum++; | |
async.retry((retryCb) => { | |
s3.uploadPart({ | |
Body: fileData.slice(rangeStart, end), | |
Bucket: bucketName, | |
Key: fileName, | |
PartNumber: partNum, | |
UploadId: multipart.UploadId | |
}, (err, mData) => { | |
retryCb(err, mData); | |
}); | |
}, (err, data) => { | |
//console.log(data); | |
next(err, {ETag: data.ETag, PartNumber: partNum}); | |
}); | |
}, (err, dataPacks) => { | |
s3.completeMultipartUpload({ | |
Bucket: bucketName, | |
Key: fileName, | |
MultipartUpload: { | |
Parts: dataPacks | |
}, | |
UploadId: multipart.UploadId | |
}, uploadCb); | |
}); | |
}); | |
}else{ | |
uploadCb(mpErr); | |
} | |
}); | |
} | |
function uploadFile(absoluteFilePath, uploadCb) { | |
var fileName = path.basename(absoluteFilePath); | |
var stats = fs.statSync(absoluteFilePath) | |
var fileSizeInBytes = stats["size"] | |
if(fileSizeInBytes < (1024*1024*5)) { | |
async.retry((retryCb) => { | |
fs.readFile(absoluteFilePath, (err, fileData) => { | |
s3.putObject({ | |
Bucket: bucketName, | |
Key: fileName, | |
Body: fileData | |
}, retryCb); | |
}); | |
}, uploadCb); | |
}else{ | |
uploadMultipart(absoluteFilePath, fileName, uploadCb) | |
} | |
} | |
This works fine in my end. Thanks a lot!
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property
Does the SDK handle this for us now or no?