AWS EBS Instance Autostart

README.md

License

Apache 2.0 No warranty

Summary

This Terraform automates the deployment of a Cloudwatch Event Rule based auto-recovery system that allows nodes/instances to be automatically restarted in case they are shutdown. The shutdown could have been triggered as a part of AWS scheduled host maintenance or due to accidental shutdown of an instance.

Cloudwatch Event Rule monitors EC2 state change events and triggers Lambda in case that event is an "Instance Shutdown" event. Once triggered the Lambda will then call the AWS EC2 APIs and start that instance back up again.

This system is specifically benefitial when using EBS backed storage however, it can also be useful for non-EBS storage types at the cost of data loss.

How to use?

Pre-requisites:

1. Installed AWS CLI, Terraform

2. AWS credentials with permissions to operate on IAM (create/attach/apply) role, policy), Lambda and Cloudwatch

Steps

1. Zip the function.js file to function.zip file using zip function.zip function.js

2. Run terraform apply

node-recovery.tf

# Terraform template to have AWS EBS Node auto restart when shutdown

provider "aws" {
  region = "us-east-1"
}

resource "aws_iam_role" "node_recovery_lambda_role" {
  name = "node_recovery_lambda_role"
  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
    "Statement": [
      {
        "Action": "sts:AssumeRole",
        "Principal": {
          "Service": "lambda.amazonaws.com"
        },
        "Effect": "Allow"
      }
    ]
}
EOF
}

resource "aws_iam_policy" "standard_lambda_policy" {
  name = "standard_lambda_policy"
  policy = <<EOF
{
    "Version": "2012-10-17",
    "Statement": [
      {
        "Action": [
          "logs:CreateLogGroup",
          "logs:CreateLogStream",
          "logs:PutLogEvents"
        ],
        "Effect": "Allow",
        "Resource": "*"
      }
    ]
}
EOF
}

resource "aws_iam_policy" "node_recovery_ec2_lambda_policy" {
  name = "node_recovery_ec2_lambda_policy"
  policy = <<EOF
{
      "Version": "2012-10-17",
      "Statement": [
          {
              "Sid": "StmtEc2LambdaPolicy",
              "Effect": "Allow",
              "Action": [
                  "ec2:StartInstances"
              ],
              "Resource": [
                  "arn:aws:ec2:us-east-1:"accountid":instance/*"
              ]
          }
      ]
}
EOF
}

resource "aws_iam_role_policy_attachment" "lambda_ec2_policy" {
  role = "${aws_iam_role.node_recovery_lambda_role.name}"
  policy_arn = "${aws_iam_policy.node_recovery_ec2_lambda_policy.arn}"
}

resource "aws_iam_role_policy_attachment" "lambda_lambda_policy" {
  role = "${aws_iam_role.node_recovery_lambda_role.name}"
  policy_arn = "${aws_iam_policy.standard_lambda_policy.arn}"
}


resource "aws_cloudwatch_event_rule" "node_recovery" {
  name        = "lambda_node_recovery"
  description = "Trigger automated node recovery"

  event_pattern = <<PATTERN
{
    "source": [
      "aws.ec2"
    ],
    "detail-type": [
      "EC2 Instance State-change Notification"
    ],
    "detail": {
      "state": [
        "stopped"
      ]
    }
}
PATTERN
}

resource "aws_lambda_function" "node_recovery_lambda" {
  filename         = "node-recovery/target/node-recovery-0.0.1-SNAPSHOT.jar"
  function_name    = "node_recovery_lambda"
  role             = "${aws_iam_role.node_recovery_lambda_role.arn}"
  handler          = "com.srotya.lambda.noderecovery.NodeRecovery"
  source_code_hash = "${base64sha256(file("node-recovery/target/node-recovery-0.0.1-SNAPSHOT.jar"))}"
  runtime          = "java8"
  memory_size      = 512
  timeout          = 10
  environment {
    variables = {
      foo = "bar"
    }
  }
}

resource "aws_lambda_alias" "node_recovery_lambda" {
  name             = "node_recovery_lambda"
  description      = "a sample description"
  function_name    = "${aws_lambda_function.node_recovery_lambda.arn}"
  function_version = "$LATEST"
}

resource "aws_lambda_permission" "allow_cloudwatch" {
  statement_id   = "AllowExecutionFromCloudWatch"
  action         = "lambda:InvokeFunction"
  function_name  = "${aws_lambda_function.node_recovery_lambda.function_name}"
  principal      = "events.amazonaws.com"
  source_arn     = "${aws_cloudwatch_event_rule.node_recovery.arn}"
}

resource "aws_cloudwatch_event_target" "node_recovery_lambda_target" {
  rule      = "${aws_cloudwatch_event_rule.node_recovery.name}"
  target_id = "SendToLambda"
  arn       = "${aws_lambda_function.node_recovery_lambda.arn}"
}

NodeRecovery.java

package com.srotya.lambda.noderecovery;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.AmazonEC2ClientBuilder;
import com.amazonaws.services.ec2.model.StartInstancesRequest;
import com.amazonaws.services.ec2.model.StartInstancesResult;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;

/**
 * @author ambud.sharma
 */
public class NodeRecovery implements RequestHandler<Map<String, Object>, Void> {

	@Override
	public Void handleRequest(Map<String, Object> event, Context ctx) {
		@SuppressWarnings("unchecked")
		Map<String, Object> map = (Map<String, Object>) event.get("detail");
		String instanceId = (String) map.get("instance-id");
		System.out.println("Attempting to start Instance ID:" + instanceId);
		AWSCredentialsProvider creds = new DefaultAWSCredentialsProviderChain();
		AmazonEC2ClientBuilder builder = AmazonEC2ClientBuilder.standard();
		builder.setCredentials(creds);
		AmazonEC2 ec2 = builder.build();
		StartInstancesRequest startInstancesRequest = new StartInstancesRequest();
		startInstancesRequest.setInstanceIds(Arrays.asList(instanceId));
		StartInstancesResult result = ec2.startInstances(startInstancesRequest);
		System.out.println("Instance ID:" + instanceId + " start result:" + result.getStartingInstances());
		return null;
	}

	public static void main(String[] args) {
		NodeRecovery recovery = new NodeRecovery();
		Map<String, Object> event = new HashMap<>();
		Map<String, Object> detail = new HashMap<>();
		detail.put("instance-id", "i-2323424234324");
		event.put("detail", detail);
		recovery.handleRequest(event, null);
	}

}

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<groupId>com.srotya.lambda</groupId>
	<artifactId>node-recovery</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<packaging>jar</packaging>

	<name>node-recovery</name>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
	</properties>

	<dependencies>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>4.12</version>
			<scope>test</scope>
		</dependency>
		<dependency>
			<groupId>com.amazonaws</groupId>
			<artifactId>aws-lambda-java-core</artifactId>
			<version>1.1.0</version>
		</dependency>
		<dependency>
			<groupId>com.amazonaws</groupId>
			<artifactId>aws-java-sdk-ec2</artifactId>
			<version>1.11.113</version>
		</dependency>
	</dependencies>
	<build>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>3.5.1</version>
				<configuration>
					<source>1.8</source>
					<target>1.8</target>
				</configuration>
			</plugin>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-shade-plugin</artifactId>
				<version>2.3</version>
				<configuration>
					<createDependencyReducedPom>false</createDependencyReducedPom>
					<filters>
						<filter>
							<artifact>*:*</artifact>
							<excludes>
								<exclude>META-INF/*.SF</exclude>
								<exclude>META-INF/*.DSA</exclude>
								<exclude>META-INF/*.RSA</exclude>
							</excludes>
						</filter>
					</filters>
				</configuration>
				<executions>
					<execution>
						<goals>
							<goal>shade</goal>
						</goals>
						<phase>package</phase>
						<configuration>
							<transformers>
								<transformer
									implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
							</transformers>
						</configuration>
					</execution>
				</executions>
			</plugin>
		</plugins>
	</build>
</project>