Skip to content

Instantly share code, notes, and snippets.

@mickymots
Created March 10, 2021 15:27
Show Gist options
  • Save mickymots/f37d4a8294d1250d9a0fb84ab4356b1b to your computer and use it in GitHub Desktop.
Save mickymots/f37d4a8294d1250d9a0fb84ab4356b1b to your computer and use it in GitHub Desktop.
// dependencies
const AWS = require('aws-sdk');
const util = require('util');
// const attr = require('dynamodb-data-types').AttributeValue;
// get reference to S3 client
const s3 = new AWS.S3();
// Set the AWS Region
const REGION = "ca-central-1"; //e.g. "us-east-1"
const dbclient = new AWS.DynamoDB.DocumentClient({ region: REGION });
// JSON - Insert to Dynamo Table
const insertToDynamoTable = async function(json){
try{
let dynamoTableName = 'sample-table'
let dynamoDBRecords = getDynamoDBRecords(json)
let batches = [];
while(dynamoDBRecords.length) {
batches.push(dynamoDBRecords.splice(0, 25));
}
await Promise.all(
batches.map(async (batch) =>{
let requestItems = {}
requestItems[dynamoTableName] = batch
const params = {
RequestItems: requestItems
};
console.log(params)
await dbclient.batchWrite(params).promise()
})
);
} catch (error) {
console.log(error);
return error;
}
}
// Get DynamoDB records from json
const getDynamoDBRecords = function (data) {
let dynamoDBRecords = data.Sheet1.map(entity => {
console.log(entity)
let dynamoRecord = Object.assign({ PutRequest: { Item: entity } })
return dynamoRecord
})
return dynamoDBRecords
}
exports.handler = async (event, context) => {
// Read options from the event parameter.
console.log("Reading options from event:\n", util.inspect(event, { depth: 5 }));
const srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
const srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
let json_data = {}
let response = {}
try {
const params = {
Bucket: srcBucket,
Key: srcKey
};
response = await s3.getObject(params).promise();
} catch (error) {
console.log(error);
return;
}
json_data = response.Body.toString('utf-8')
json_data = JSON.parse(json_data)
const data = await insertToDynamoTable(json_data)
return `Successfully processed ${json_data.Sheet1.length} records.`;
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment