continuous deployment of Golang Gin application in AWS Lambda and Amazon API Gateway with CodePipeline/CodeBuild

README.md

You can use CodePipeline and CodeBuild to create a continuous deployment/integration pipeline for serverless applications build on AWS Lambda and Amazon API Gateway. This sample application is written in Go with the Gin framework and uses the eawsy API Gateway proxy shim: https://github.com/eawsy/aws-lambda-go-net

We initially detailed our methodology in this blog post: https://aws.amazon.com/blogs/compute/continuous-deployment-for-serverless-applications/

We have used the shim technology created by eawsy to run Golang applications inside AWS Lambda (https://github.com/eawsy/aws-lambda-go-shim) and created a container that can be used with CodeBuild as part of our original pipeline template.

The container is available on DockerHub and is called sapessi/aws-lambda-go18-codebuild:latest. To use this container, simply change the Image property of the CodeBuild project environment.

The pipeline template, sample app, buildspec and SAM files are attached to this gist.

app-sam.yaml

AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31

Resources:
  TestFunction:
    Type: AWS::Serverless::Function
    Properties:
      Handler: handler.Handle
      CodeUri: /tmp/package.zip
      Runtime: python2.7
      MemorySize: 128
      Timeout: 20
      Events:
        ProxyResource:
          Type: Api
          Properties:
            Path: /{proxy+}
            Method: ANY

Outputs:
  ApiUrl:
    Description: URL for application
    Value: !Sub 'https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod'

buildspec.yml

version: 0.1

phases:
  install:
    commands:
      - go get gopkg.in/gin-gonic/gin.v1
  pre_build:
    commands:
      - rm -rf /tmp/package.zip handler.so
  build:
    commands:
      - mkdir dist
      - export GOPATH=$GOPATH:$CODEBUILD_SRC_DIR/models; cd your_service/src; go build -buildmode=plugin -ldflags='-w -s -pluginpath lambda' -o /dist/handler.so
      - cd /dist; /shim/pack handler handler.so package.zip
      - mv /dist/package.zip /tmp/package.zip
  post_build:
    commands:
      - aws cloudformation package --template-file app-sam.yaml --s3-bucket $BUILD_OUTPUT_BUCKET --output-template-file app-output_sam.yaml
      - aws cloudformation package --template-file test-sam.yaml --s3-bucket $BUILD_OUTPUT_BUCKET --output-template-file test-output_sam.yaml

artifacts:
  files:
    - app-output_sam.yaml
    - test-output_sam.yaml
  discard-paths: yes
~                    

pipeline.template

AWSTemplateFormatVersion: 2010-09-09
Parameters:
    ServiceName:
        Type: String
        Default: RestaurantService
        Description: Name for the service, used in the code repository, Lambda function, and pipeline names
    CodeBuildEnvironment:
        Type: String
        Default: "sapessi/aws-lambda-go18-codebuild:latest"
        Description: Name of the image to use for the CodeBuild container
Resources:
    # Code repository for the service
    CodeRepository:
        Type: AWS::CodeCommit::Repository
        Properties:
            RepositoryName: !Sub '${ServiceName}_repo'
            RepositoryDescription: !Sub 'Repository for the ${ServiceName} service'


    # CodeBuild project and resources (S3 Bucket for build artifacts, Role, Project)
    BuildArtifactsBucket:
        Type: AWS::S3::Bucket
    CodeBuildServiceRole:
        Type: AWS::IAM::Role
        Properties:
            AssumeRolePolicyDocument:
                Version: '2012-10-17'
                Statement:
                  - Action: 
                      - 'sts:AssumeRole'
                    Effect: Allow
                    Principal:
                      Service:
                        - codebuild.amazonaws.com
            Path: /
            Policies:
                - PolicyName: CodeBuildAccess
                  PolicyDocument:
                    Version: '2012-10-17'
                    Statement:
                      - Effect: Allow
                        Resource:
                          - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ServiceName}_build'
                          - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ServiceName}_build:*'
                        Action:
                          - 'logs:CreateLogGroup'
                          - 'logs:CreateLogStream'
                          - 'logs:PutLogEvents'
                      - Effect: Allow
                        Resource:
                          - !Sub 'arn:aws:s3:::${BuildArtifactsBucket}/*'
                        Action: 
                          - 's3:GetObject'
                          - 's3:GetObjectVersion'
                          - 's3:PutObject'
    CodeBuildProject:
        Type: AWS::CodeBuild::Project
        Properties:
            Name: !Sub '${ServiceName}_build'
            Artifacts:
                Type: CODEPIPELINE
            Environment: 
                Type: LINUX_CONTAINER
                ComputeType: BUILD_GENERAL1_SMALL
                Image: !Sub '${CodeBuildEnvironment}'
                EnvironmentVariables:
                  - Name: BUILD_OUTPUT_BUCKET
                    Value: !Ref BuildArtifactsBucket
            ServiceRole: !GetAtt CodeBuildServiceRole.Arn
            Source: 
                Type: CODEPIPELINE
    
    # Integration tests function
    StartTestsFunctionExecutionRole:
      Type: AWS::IAM::Role
      Properties:
        AssumeRolePolicyDocument:
          Version: '2012-10-17'
          Statement:
            - Action: 
                - 'sts:AssumeRole'
              Effect: Allow
              Principal:
                Service: 
                  - lambda.amazonaws.com
        Path: /
        Policies:
          - PolicyName: InvokeAsync
            PolicyDocument:
              Version: '2012-10-17'
              Statement:
                - Action:
                    - 'lambda:InvokeAsync'
                    - 'lambda:InvokeFunction'
                    - 'cloudformation:DescribeStacks'
                    - 'codepipeline:*'
                    - 'logs:*'
                  Effect: Allow
                  Resource: '*'

    StartTestsFunction:
      Type: AWS::Lambda::Function
      Properties:
        FunctionName: !Sub '${ServiceName}_start_tests'
        Description: !Sub 'Starts integration tests function ${ServiceName}'
        Handler: index.lambda_handler
        MemorySize: 128
        Role: !GetAtt StartTestsFunctionExecutionRole.Arn
        Runtime: python2.7
        Timeout: 10
        Code:
          ZipFile: >
            import json

            import boto3

            import uuid

            cp = boto3.client("codepipeline")

            cf = boto3.client("cloudformation")

            lambda_client = boto3.client("lambda")

            def lambda_handler(event, context):
                print json.dumps(event, sort_keys=True, indent=4, separators=(',', ': '))

                job_id = event["CodePipeline.job"]["id"] 
                user_parameters = event["CodePipeline.job"]["data"]["actionConfiguration"]["configuration"]["UserParameters"]
                stack_name = user_parameters.split("|")[0]
                test_stack_name = user_parameters.split("|")[1]
                stage = user_parameters.split("|")[2]

                print("looking for ApiUrl in {} Stack".format(stack_name))
                api_url, error = _get_property_from_stack(stack_name, "ApiUrl")
                if not api_url:
                    _fail_job(job_id, error)
                    return

                print("looking for TestFunction in {} Stack".format(test_stack_name))
                test_function, error = _get_property_from_stack(test_stack_name, "TestFunction")
                if not test_function:
                    _fail_job(job_id, error)
                    return

                lambda_client.invoke_async(
                  FunctionName=test_function,
                  InvokeArgs=json.dumps({
                    "job_id": job_id,
                    "api_url": api_url
                  })
                )

            def _get_property_from_stack(stack_name, property_key):
                stack_data = cf.describe_stacks(StackName=stack_name)

                if not stack_data or len(stack_data["Stacks"]) == 0:
                    return None, "Could not find stack"
                    
                current_stack = stack_data["Stacks"][0]

                if "Outputs" not in current_stack:
                    return None, "Could not find outputs in stack"
                    
                for cur_output in current_stack["Outputs"]:
                    if cur_output["OutputKey"] == property_key:
                        return cur_output["OutputValue"], None
                
                return None, "Could not find {} output property in stack".format(property_key)

            def _complete_job(job_id):
                cp.put_job_success_result(jobId=job_id)

            def _fail_job(job_id, failure_reason):
                cp.put_job_failure_result(
                        jobId=job_id,
                        failureDetails={
                            'type': 'JobFailed',
                            'message': failure_reason,
                            'externalExecutionId': uuid.uuid4().hex
                        }
                    )
    
    # CodePipeline definition and required roles
    CFNPipelinePolicy:
        Type: AWS::IAM::ManagedPolicy
        Properties:
            Description: CloudFormation Pipeline Execution Policy
            Path: "/"
            PolicyDocument:
                Version: '2012-10-17'
                Statement:
                    Effect: Allow
                    Action: 
                        - 'cloudformation:CreateStack'
                        - 'cloudformation:DescribeStacks'
                        - 'cloudformation:DeleteStack'
                        - 'cloudformation:UpdateStack'
                        - 'cloudformation:CreateChangeSet'
                        - 'cloudformation:ExecuteChangeSet'
                        - 'cloudformation:DeleteChangeSet'
                        - 'cloudformation:DescribeChangeSet'
                        - 'cloudformation:SetStackPolicy'
                        - 'cloudformation:SetStackPolicy'
                        - 'cloudformation:ValidateTemplate'
                        - 'codebuild:StartBuild'
                        - 'codebuild:BatchGetBuilds'
                    Resource: "*"
    CloudFormationExecutionRole:
      Type: AWS::IAM::Role
      Properties:
        AssumeRolePolicyDocument:
          Version: '2012-10-17'
          Statement:
            Action: 
              - 'sts:AssumeRole'
            Effect: Allow
            Principal:
              Service:
                - cloudformation.amazonaws.com
        Path: /
        ManagedPolicyArns:
          - 'arn:aws:iam::aws:policy/AdministratorAccess'
    PipelineExecutionRole:
        Type: AWS::IAM::Role
        Properties:
            AssumeRolePolicyDocument:
                Version: '2012-10-17'
                Statement:
                  - Action: 
                    - 'sts:AssumeRole'
                    Effect: Allow
                    Principal:
                      Service: 
                        - codepipeline.amazonaws.com
            Path: /
            ManagedPolicyArns:
                - 'arn:aws:iam::aws:policy/AWSCodeCommitFullAccess'
                - 'arn:aws:iam::aws:policy/AmazonS3FullAccess'
                - !Ref CFNPipelinePolicy
            Policies:
                - PolicyName: CodePipelineAccess
                  PolicyDocument:
                    Version: '2012-10-17'
                    Statement:
                        - Action:
                            - 'iam:PassRole'
                            - 'lambda:InvokeFunction'
                            - 'lambda:ListFunctions'
                            - 'lambda:InvokeAsyc'
                          Effect: Allow
                          Resource: '*'
    Pipeline:
        Type: AWS::CodePipeline::Pipeline
        Properties:
            ArtifactStore: 
                Location: !Ref BuildArtifactsBucket
                Type: S3
            Name: !Sub ${ServiceName}_pipeline
            RoleArn: !GetAtt PipelineExecutionRole.Arn
            Stages:
                - Name: Source
                  Actions:
                    - Name: CodeCommitRepo
                      ActionTypeId:
                        Category: Source
                        Owner: AWS
                        Provider: CodeCommit
                        Version: 1
                      Configuration:
                        RepositoryName: !Sub '${ServiceName}_repo'
                        BranchName: master
                      OutputArtifacts:
                        - Name: SourceZip
                      RunOrder: 1
                - Name: Build
                  Actions:
                    - Name: CodeBuild
                      ActionTypeId:
                        Category: Build
                        Owner: AWS
                        Provider: CodeBuild
                        Version: 1
                      Configuration:
                        ProjectName: !Ref CodeBuildProject
                      InputArtifacts:
                        - Name: SourceZip
                      OutputArtifacts:
                        - Name: BuiltZip
                - Name: DeployTests
                  Actions:
                    - Name: CreateChangeSet
                      ActionTypeId:
                        Category: Deploy
                        Owner: AWS
                        Provider: CloudFormation
                        Version: 1
                      Configuration:
                        ActionMode: CHANGE_SET_REPLACE
                        RoleArn: !GetAtt CloudFormationExecutionRole.Arn
                        StackName: !Sub '${ServiceName}-Stack-Tests'
                        ChangeSetName: !Sub '${ServiceName}-ChangeSet-Tests'
                        TemplatePath: BuiltZip::test-output_sam.yaml
                        Capabilities: CAPABILITY_IAM
                      InputArtifacts:
                        - Name: BuiltZip
                      OutputArtifacts:
                        - Name: TestChangeSet
                      RunOrder: 1
                    - Name: ExecuteChangeSet
                      ActionTypeId:
                        Category: Deploy
                        Owner: AWS
                        Provider: CloudFormation
                        Version: 1
                      Configuration:
                        ActionMode: CHANGE_SET_EXECUTE
                        RoleArn: !GetAtt CloudFormationExecutionRole.Arn
                        StackName: !Sub '${ServiceName}-Stack-Tests'
                        ChangeSetName: !Sub '${ServiceName}-ChangeSet-Tests'
                      OutputArtifacts:
                        - Name: ExecutedTestChangeSet
                      RunOrder: 2
                - Name: Beta
                  Actions:
                    - Name: CreateChangeSet
                      ActionTypeId:
                        Category: Deploy
                        Owner: AWS
                        Provider: CloudFormation
                        Version: 1
                      Configuration:
                        ActionMode: CHANGE_SET_REPLACE
                        RoleArn: !GetAtt CloudFormationExecutionRole.Arn
                        StackName: !Sub '${ServiceName}-Stack-Beta'
                        ChangeSetName: !Sub '${ServiceName}-ChangeSet-Beta'
                        TemplatePath: BuiltZip::app-output_sam.yaml
                        Capabilities: CAPABILITY_IAM
                      InputArtifacts:
                        - Name: BuiltZip
                      RunOrder: 1
                    - Name: ExecuteChangeSet
                      ActionTypeId:
                        Category: Deploy
                        Owner: AWS
                        Provider: CloudFormation
                        Version: 1
                      Configuration:
                        ActionMode: CHANGE_SET_EXECUTE
                        RoleArn: !GetAtt CloudFormationExecutionRole.Arn
                        StackName: !Sub '${ServiceName}-Stack-Beta'
                        ChangeSetName: !Sub '${ServiceName}-ChangeSet-Beta'
                      OutputArtifacts:
                        - Name: !Sub '${ServiceName}BetaChangeSet'
                      RunOrder: 2
                    - Name: IntegrationTests
                      ActionTypeId:
                        Category: Invoke
                        Owner: AWS
                        Provider: Lambda
                        Version: 1
                      Configuration:
                        FunctionName: !Ref StartTestsFunction
                        UserParameters: !Sub '${ServiceName}-Stack-Beta|${ServiceName}-Stack-Tests|beta'
                      InputArtifacts:
                        - Name: !Sub '${ServiceName}BetaChangeSet'
                      RunOrder: 3
                - Name: Gamma
                  Actions:
                    - Name: CreateChangeSet
                      ActionTypeId:
                        Category: Deploy
                        Owner: AWS
                        Provider: CloudFormation
                        Version: 1
                      Configuration:
                        ActionMode: CHANGE_SET_REPLACE
                        RoleArn: !GetAtt CloudFormationExecutionRole.Arn
                        StackName: !Sub '${ServiceName}-Stack-Gamma'
                        ChangeSetName: !Sub '${ServiceName}-ChangeSet-Gamma'
                        TemplatePath: BuiltZip::app-output_sam.yaml
                        Capabilities: CAPABILITY_IAM
                      InputArtifacts:
                        - Name: BuiltZip
                      RunOrder: 1
                    - Name: ExecuteChangeSet
                      ActionTypeId:
                        Category: Deploy
                        Owner: AWS
                        Provider: CloudFormation
                        Version: 1
                      Configuration:
                        ActionMode: CHANGE_SET_EXECUTE
                        RoleArn: !GetAtt CloudFormationExecutionRole.Arn
                        StackName: !Sub '${ServiceName}-Stack-Gamma'
                        ChangeSetName: !Sub '${ServiceName}-ChangeSet-Gamma'
                      OutputArtifacts:
                        - Name: !Sub '${ServiceName}GammaChangeSet'
                      RunOrder: 2
                    - Name: IntegrationTests
                      ActionTypeId:
                        Category: Invoke
                        Owner: AWS
                        Provider: Lambda
                        Version: 1
                      Configuration:
                        FunctionName: !Ref StartTestsFunction
                        UserParameters: !Sub '${ServiceName}-Stack-Gamma|${ServiceName}-Stack-Tests|gamma'
                      InputArtifacts:
                        - Name: !Sub '${ServiceName}GammaChangeSet'
                      RunOrder: 3
                - Name: Prod
                  Actions:
                    - Name: DeploymentApproval
                      ActionTypeId:
                        Category: Approval
                        Owner: AWS
                        Provider: Manual
                        Version: 1
                      RunOrder: 1
                    - Name: CreateChangeSet
                      ActionTypeId:
                        Category: Deploy
                        Owner: AWS
                        Provider: CloudFormation
                        Version: 1
                      Configuration:
                        ActionMode: CHANGE_SET_REPLACE
                        RoleArn: !GetAtt CloudFormationExecutionRole.Arn
                        StackName: !Sub '${ServiceName}-Stack-Prod'
                        ChangeSetName: !Sub '${ServiceName}-ChangeSet-Prod'
                        TemplatePath: BuiltZip::app-output_sam.yaml
                        Capabilities: CAPABILITY_IAM
                      InputArtifacts:
                        - Name: BuiltZip
                      RunOrder: 2
                    - Name: ExecuteChangeSet
                      ActionTypeId:
                        Category: Deploy
                        Owner: AWS
                        Provider: CloudFormation
                        Version: 1
                      Configuration:
                        ActionMode: CHANGE_SET_EXECUTE
                        RoleArn: !GetAtt CloudFormationExecutionRole.Arn
                        StackName: !Sub '${ServiceName}-Stack-Prod'
                        ChangeSetName: !Sub '${ServiceName}-ChangeSet-Prod'
                      OutputArtifacts:
                        - Name: !Sub '${ServiceName}ProdChangeSet'
                      RunOrder: 3

service.go

package main

import (
        "net/http"

        "gopkg.in/gin-gonic/gin.v1"

        "github.com/eawsy/aws-lambda-go-net/service/lambda/runtime/net"
        "github.com/eawsy/aws-lambda-go-net/service/lambda/runtime/net/apigatewayproxy"
)

var Handle apigatewayproxy.Handler

func init() {
  r := gin.Default()

  r.GET("/users", func(c *gin.Context) {
    c.JSON(http.StatusOK, gin.H{ "message": "Hello World" })
  })

  //r.Run("localhost:8080")
  ln := net.Listen()
  Handle = apigatewayproxy.New(ln, nil).Handle
  go http.Serve(ln, r)
}

@sapessi

• import "C" statement is no more needed 😉
• default package name is now handler.zip and no more package.zip
• the content of the shim folder inside the docker images is now compiled python and no more regular python. We also added a pack python script which can assist you in injecting the shim into your final zip.

Nice, I'll update this and rebuild the container with your prod version soon.