Last active
January 25, 2023 18:30
-
-
Save johnib/14b090d0883ee2d2fb56dd3ac3c82f9b to your computer and use it in GitHub Desktop.
Copy S3 data to DynamoDB
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
let path = require('path'); | |
let aws = require('aws-sdk'); | |
let s3 = new aws.S3(); | |
let zlib = require('zlib'); | |
let docClient = new aws.DynamoDB.DocumentClient(); | |
exports.handler = (event, context, callback) => { | |
context.callbackWaitsForEmptyEventLoop = false; | |
event.Records.forEach(record => { | |
const params = { | |
Bucket: record.s3.bucket.name, | |
Key: record.s3.object.key | |
}; | |
const isGzip = path.extname(params.Key) === ".gz"; | |
let readStream = s3.getObject(params).createReadStream(); | |
readStream = isGzip ? readStream.pipe(zlib.createGunzip()) : readStream; | |
readStream.on('data', data => { | |
// transform to string | |
// seperate lines, remember that the first / last lines could be partial of the previous / next 'data' event | |
const dynamoDbDocument = { | |
// csv parse each line | |
}; | |
const dynamoDbParams = { | |
TableName: "Table Name", | |
Item: dynamoDbDocument | |
}; | |
docClient.put(params, function (err, data) { | |
if (err) { | |
console.error("Unable to add item. Error JSON:", JSON.stringify(err, null, 2)); | |
} else { | |
console.log("Added item:", JSON.stringify(data, null, 2)); | |
} | |
}); | |
}); | |
readStream.on('end', () => { | |
// there are several IO operations that need to occur before calling this callback | |
// ending the execution this way will not work. | |
// use promises and wait on all of them before ending the execution | |
callback(null, ''); | |
}); | |
}); | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment