Example AWS Lambda for converting heka logs into json

lambda.js

var Zlib = require('zlib')
var AWS = require('aws-sdk')
var HekaDecodeStream = require('heka-decode-stream')
var through = require('through2')

var s3 = new AWS.S3()

exports.handler = function (event, context) {
	var i = 0
	var record = event.Records[0].s3
	var sourceBucket = record.bucket.name
	var sourceKey = decodeURIComponent(record.object.key.replace(/\+/g, " "))
	var outputBucket = sourceBucket + '-json'
	var outputKey = 'json-' + sourceKey
	
	var output = s3.getObject{ Bucket: sourceBucket, Key: sourceKey }).createReadStream()
		.pipe(Zlib.createGunzip())
		.pipe(HekaDecodeStream({
			filter: function (m) { return m.type === 'mozsvc.metrics' }
		}))
		.pipe(through.obj(function (o, _, cb) {
			i++
			if (i % 10000 === 0) {
				console.log('record: ' + i + ' at ' + Date.now())
			}
			cb(null, JSON.stringify(o) + '\n')
		}))
		.pipe(Zlib.createGzip())
	
	s3.upload({ Bucket: outputBucket, Key: outputKey, Body: output },
		function (err, data) {
			context.done(err, data)
		})
}