Kinesis Firehose demo

kinesis-firehose-demo

# assumes a kinesis firehose delivery system is in place (e.g. demo-firehose-stream)

# NodeJS Lambda function to append CRs as follows with a timeout > 1 min
# Configure this to deposit results in S3

console.log('Loading function');

exports.handler = async (event, context) => {
    
    /* Process the list of records and transform them */
    const output = event.records.map((record) => {
        let entry = (new Buffer(record.data, 'base64')).toString('utf8');
        let result = entry + '\n';
        const payload = (new Buffer(result, 'utf8')).toString('base64');
        return {
            recordId: record.recordId,
            result: 'Ok',
            data: payload
        }
    });
    console.log(`Processing completed.  Successful records ${output.length}.`);
    return { records: output };
};

# Use the following CLI commands to put records into the firehose

aws firehose put-record --delivery-stream-name demo-forehose-stream --record Data=100
aws firehose put-record --delivery-stream-name demo-forehose-stream --record Data=101
aws firehose put-record --delivery-stream-name demo-forehose-stream --record Data=102
...