My work so far on build cache using lambda and DynamoDB

index.js

'use strict'

console.log('Loading function')

var aws = require('aws-sdk')
var async = require('async')
var doc = require('dynamodb-doc')
var dynamo = new doc.DynamoDB()
var s3 = new aws.S3()

var internals = {}

internals.tableName = 'buildkite-builds'
internals.indexName = 'project_id-updated_at-index'
internals.limit = 25

internals.bucketName = 'XX.wolfe.id.au'
internals.buildsFileName = 'builds.json'

// requires a random string to be included, this is provided by the buildkite Settings page under Notifications -> Webhook Notifications
internals.validToken = 'XXX'
internals.validEventTypes = ['build.scheduled', 'build.running', 'build.finished']

exports.handler = function (event, context) {
  // console.log('Received event:', JSON.stringify(event, null, 2))
  async.waterfall([
        async.apply(checkToken, event),
        checkEventType,
        transformEvent,
        saveBuild,
        retrieveLatestBuilds,
        uploadToS3
  ], context.done)
}

function checkToken (event, cb) {
  // check the token, because yeah this is the future of security..
  if (event.buildkiteToken !== internals.validToken) {
    cb(new Error('Unrecognized token in event'))
  } else {
    cb(null, event)
  }
}

function checkEventType (event, cb) {
  if (internals.validEventTypes.indexOf(event.buildkiteEvent) !== -1) {
    cb(null, event)
  } else {
    cb(new Error('Unrecognized buildkite event: "' + event.buildkiteEvent + '"'))
  }
}

// rejig the event
function transformEvent (event, cb) {

  var build = event.build
  var project = event.project

  // flatten some stuff
  build.project_id = project.id
  build.project_web_url = project.web_url
  build.project_name = project.name
  build.creator_name = build.creator.name
  build.creator_email = build.creator.email

  // clear out cruft
  delete build.creator

  build.updated_at = Date.now()

  cb(null, build)
}

// convert the table and save the build into a dynamodb
function saveBuild (build, cb) {
  dynamo.putItem({
    TableName: internals.tableName,
    Item: build
  }, cb)
}

function retrieveLatestBuilds (build, cb) {
  dynamo.scan({
    Limit: internals.limit,
    TableName: internals.tableName
  }, cb)
}

function uploadToS3 (builds, cb) {
  s3.putObject({
    Bucket: internals.bucketName,
    Key: internals.buildsFileName,
    Body: JSON.stringify(builds),
    ACL: 'public-read'
  }, cb)
}