Created
March 10, 2021 15:27
-
-
Save mickymots/f37d4a8294d1250d9a0fb84ab4356b1b to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// dependencies | |
const AWS = require('aws-sdk'); | |
const util = require('util'); | |
// const attr = require('dynamodb-data-types').AttributeValue; | |
// get reference to S3 client | |
const s3 = new AWS.S3(); | |
// Set the AWS Region | |
const REGION = "ca-central-1"; //e.g. "us-east-1" | |
const dbclient = new AWS.DynamoDB.DocumentClient({ region: REGION }); | |
// JSON - Insert to Dynamo Table | |
const insertToDynamoTable = async function(json){ | |
try{ | |
let dynamoTableName = 'sample-table' | |
let dynamoDBRecords = getDynamoDBRecords(json) | |
let batches = []; | |
while(dynamoDBRecords.length) { | |
batches.push(dynamoDBRecords.splice(0, 25)); | |
} | |
await Promise.all( | |
batches.map(async (batch) =>{ | |
let requestItems = {} | |
requestItems[dynamoTableName] = batch | |
const params = { | |
RequestItems: requestItems | |
}; | |
console.log(params) | |
await dbclient.batchWrite(params).promise() | |
}) | |
); | |
} catch (error) { | |
console.log(error); | |
return error; | |
} | |
} | |
// Get DynamoDB records from json | |
const getDynamoDBRecords = function (data) { | |
let dynamoDBRecords = data.Sheet1.map(entity => { | |
console.log(entity) | |
let dynamoRecord = Object.assign({ PutRequest: { Item: entity } }) | |
return dynamoRecord | |
}) | |
return dynamoDBRecords | |
} | |
exports.handler = async (event, context) => { | |
// Read options from the event parameter. | |
console.log("Reading options from event:\n", util.inspect(event, { depth: 5 })); | |
const srcBucket = event.Records[0].s3.bucket.name; | |
// Object key may have spaces or unicode non-ASCII characters. | |
const srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " ")); | |
let json_data = {} | |
let response = {} | |
try { | |
const params = { | |
Bucket: srcBucket, | |
Key: srcKey | |
}; | |
response = await s3.getObject(params).promise(); | |
} catch (error) { | |
console.log(error); | |
return; | |
} | |
json_data = response.Body.toString('utf-8') | |
json_data = JSON.parse(json_data) | |
const data = await insertToDynamoTable(json_data) | |
return `Successfully processed ${json_data.Sheet1.length} records.`; | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment