Cloudformation and code to create an AWS Lambda to start a jenkins job on github push (via SNS)

README.md

Github/Jenkins Hook with AWS Lambda

Setup

• Run github_hook_cf.py to generate the cloudformation template (requires troposphere).
• Run the cloudformation template in desired region
• Once complete, take the values of the outputs and register in Github (Settings -> Webhooks & services -> Amazon SNS), this can be done on a repo or organization level
• Add the created SNS topic to the generated lambdas event source¹
• Go to the Advanced Settings settings in the lambda configuration and add it to the VPC if needed.²

Notes

1. When this cloudformation was defined, SNS was not a valid event source in the cloudformation spec
2. Lambdas were only recently made VPC compatible so this was also not settable in cloudformation

github_hook.js

console.info('Loading function');
// Libs
var http = require('http');
var qs = require('querystring');

// Config
var WATCHED_BRANCHES = ".*(" + "{{ref: WatchedBranches}}".replace(/,/g,"|") + ").*";

var REF_REGEX = "refs/heads/([^/]+)/?(.*)";

exports.handler = function(event, context) {
  // parse github event
  evt = JSON.parse(event.Records[0].Sns.Message);
  console.log(JSON.stringify(evt,null,'  '));
    
  // skip ignorable events
  if (evt.deleted) { context.fail("delete event"); return }
  if (evt.comment) { context.fail("comment event"); return }
  if (evt.ref_type=="repository") { context.fail("repo event"); return }
  if (evt.ref_type=="tag") { context.fail("tag event"); return }
  if (!evt.ref) { context.fail("missing ref"); return }
  if (!(evt.repository && evt.repository.name)) { context.fail("missing repo"); return }
  if (!(evt.ref.match(WATCHED_BRANCHES))) { context.fail("unwatched branch"); return }
  
  // Build job
  match = evt.ref.match(REF_REGEX);
  job = { repo: evt.repository.name, branch: match[1], params: {} };
  if (match[2]) 
    job.params.branch = match[2];
  console.log("Extracted job from event: ", JSON.stringify(job, null, '  '));
  
  var postdata = qs.stringify(job.params);
  var opts = {
    hostname: "{{ref: JenkinsUrl}}",
    protocol: "{{ref: JenkinsProtocol}}:".toLowerCase(),
    port: parseInt("{{ref: JenkinsPort}}"),
    path: "/job/"+job.repo+"/job/"+job.branch+"/buildWithParameters",
    method: 'POST',
    headers: {
      'Content-Type': 'application/x-www-form-urlencoded',
      'Content-Length': Buffer.byteLength(postdata)
    }
  };

  console.log("Request:", JSON.stringify(opts, null, '  '));
  var req = http.request(opts, function(res) {
    msg = job.repo + " " + job.branch + " built successfully";
    console.info(msg);
    context.succeed(msg);
  });

  req.on("error", function(e) {
    console.error("Problem reaching jenkins server");
    context.fail("Problem reaching jenkins server");
  });

  req.write(postdata);
  req.end();
};

github_hook_cf.py

#!/usr/bin/env python
import re

from troposphere import GetAtt, Join, Ref
from troposphere import Parameter, Output, Template
from troposphere import AWS_REGION

import troposphere.awslambda as awslambda
import troposphere.iam as iam
import troposphere.sns as sns

t = Template()
t.add_description("Github Hook -> SNS -> Lambda -> Jenkins REST api")

### Parameters
params = {}
params["JenkinsUrl"] = t.add_parameter(Parameter(
    "JenkinsUrl",
    Type="String",
    AllowedPattern="[a-zA-Z0-9][a-zA-Z0-9-.]*[a-zA-Z0-9]"
))

params["JenkinsProtocol"] = t.add_parameter(Parameter(
    "JenkinsProtocol",
    Type="String",
    Default="HTTP",
    AllowedValues=['HTTP','HTTPS']
))

params["JenkinsPort"] = t.add_parameter(Parameter(
    "JenkinsPort",
    Type="Number",
    Default="80",
    MinValue=1,
    MaxValue=65535
))

params["WatchedBranches"] = t.add_parameter(Parameter(
    "WatchedBranches",
    Type="CommaDelimitedList",
    Default="master, develop, feature, hotfix, release"
))
# vpc
# subnets

### Resources
## SNS
topic = t.add_resource(sns.Topic("GithubHookTopic", DisplayName="GithubHook"))

## IAM
# Github user
githubUser = t.add_resource(iam.User(
    "GithubUser",
    Policies=[iam.Policy(
        PolicyName="GithubUserPolicy",
        PolicyDocument={
          "Version": "2012-10-17",
          "Statement": [
            { "Action": [ "sns:Publish" ],
              "Resource": [ Ref(topic) ],
              "Effect": "Allow" }]})]))

githubKeys = t.add_resource(iam.AccessKey(
    "GithubKeys", 
    Status="Active",
    UserName = Ref(githubUser)))

# Lambda role
lambdaRole = t.add_resource(iam.Role(
    "GithubHookRole",
    AssumeRolePolicyDocument={
        "Version": "2012-10-17",
        "Statement": [{
            "Action": "sts:AssumeRole",
            "Principal": { "Service": "lambda.amazonaws.com" },
            "Effect": "Allow"
        }]
    },
    # NOTE: this role is a little to permissive, find ways to lock it down
    # copied from example execution policy
    Policies=[iam.Policy(
        PolicyName="GithubHookRolePolicy",
        PolicyDocument={
          "Version": "2012-10-17",
          "Statement": [ 
              { "Effect": "Allow",
                "Action": [
                    "logs:CreateLogGroup",
                    "logs:CreateLogStream",
                    "logs:PutLogEvents" ],
                "Resource": "arn:aws:logs:*:*:*" },
              { "Effect": "Allow",
                "Action": [
                    "ec2:CreateNetworkInterface",
                    "ec2:DescribeNetworkInterfaces",
                    "ec2:DetachNetworkInterface",
                    "ec2:DeleteNetworkInterface" ],
                "Resource": "*" } ] 
        })
    ]
))

## Lambda
# load code
# NOTE: line by line is less efficient but easier to validate by eye
# also, this is just a messy parser
f = open("github_hook.js", "r")
lines = []
ref_re = re.compile("\{\{(ref: [a-zA-Z][a-zA-Z0-9]*)\}\}")
for line in f.readlines():
    # shortcut if no match
    if not ref_re.search(line):
        lines.append(line)

    else:
        parts = []
        for part in ref_re.split(line):
            if part.startswith("ref: "):
                name = part[5:]
                typ = params[name].properties["Type"]

                if typ.startswith("List") or typ == "CommaDelimitedList":
                    parts.append( Join(",", Ref(name) ) )
                else:
                    parts.append( Ref(name) )

            else:
                parts.append(part)
        lines.append( Join("", parts) )

f.close()
code = Join('', lines)


# resource
lambdaFn = t.add_resource(awslambda.Function(
    "GithubHookLambda",
    Description="Consumes Github event information and triggers jenkins builds",
    Role=GetAtt(lambdaRole.title, "Arn"),
    Runtime="nodejs",
    Handler="index.handler",  # don't love this is hard coded
    MemorySize=128,     # in MB
    Timeout=10,        # in seconds
    Code=awslambda.Code(ZipFile=code)
))

### Outputs
t.add_output([
    Output("SNSTopic",
           Description="ARN for Github",
           Value=Ref(topic)),
    Output("GithubAccessKey",
           Description="Access key for github user",
           Value=Ref(githubKeys)),
    Output("GithubSecretKey",
           Description="Secret key for github user",
           Value=GetAtt(githubKeys, "SecretAccessKey")),
    Output("Region",
           Description="Region to use in github",
           Value=Ref(AWS_REGION)),
    Output("FinalSteps", Description="final steps to do", Value="1) setup data mapping in lambda, 2) add lambda to VPC if needed")
])

print t.to_json()