Accompanies http://derekdougans.blogspot.com/2016/11/running-zookeeper-and-kafka-in-aws-auto.html

user-data.sh

#!/bin/bash

# Use whichever region you're working in
export AWS_DEFAULT_REGION=eu-west-1

# Make ec2-user's home dir the install to make it easier to navigate when subsequently ssh-ing in to instances
su ec2-user -
cd /home/ec2-user

echo Working dir is `pwd`

# Use the instance metadata to find out about the network the instance has been lauched into (the IP is the standard AWS metadata IP)
AWS_METADATA_IP=169.254.169.254
MAC_ADDRESS=`curl http://$AWS_METADATA_IP/latest/meta-data/network/interfaces/macs/`
SUBNET_ID=`curl http://$AWS_METADATA_IP/latest/meta-data/network/interfaces/macs/${MAC_ADDRESS}subnet-id`
SELF_INSTANCE_ID=`curl http://$AWS_METADATA_IP/latest/meta-data/instance-id`

S3_CONFIG_BUCKET=com.example.event-servers

ENI_NAME=event-servers-eni
ENI_INDEX=1

ZK_VERSION=zookeeper-3.4.9
ZK_CONFIG_FILE=zookeeper.config

KAFKA_VERSION=kafka_2.11-0.10.0.1
KAFKA_CONFIG_FILE=kafka.config

attach_eni() {

    # Use the AWS CLI to see if the target ENI is already attached..
    CURRENT_ATTACHMENT_ID=`aws ec2 describe-network-interfaces --filters "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --query "NetworkInterfaces[0].[Attachment][0].[AttachmentId]" | grep -o 'eni-attach-[a-z0-9]*'`

    # ..and if it is, remove the attachment (the assumption here is that it's still attached to another instance that is being terminated).
    if [ $CURRENT_ATTACHMENT_ID"X" != "X" ]; then 
      echo Detaching $CURRENT_ATTACHMENT_ID;
      aws ec2 detach-network-interface --attachment-id $CURRENT_ATTACHMENT_ID
    fi

    # Use the AWS CLI to get the id of the ENI to be attached
    NETWORK_INTERFACE_ID=`aws ec2 describe-network-interfaces --filters "Name=status,Values=available" "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --output json --query "NetworkInterfaces[0].NetworkInterfaceId" | grep -o 'eni-[a-z0-9]*'`
    
    # Attach the ENI (and display the attachment id)
    echo Attaching $ENI_NAME to $ENI_INDEX  `aws ec2 attach-network-interface --network-interface-id $NETWORK_INTERFACE_ID --instance-id $SELF_INSTANCE_ID --device-index $ENI_INDEX`

} 

install_zookeeper() {

    # Copy the ZK config file from S3
    aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$ZK_CONFIG_FILE $ZK_CONFIG_FILE 

    # Format one of the EBS volumes associated with the instance and mount it for use as the ZK data directory.
    mkfs -t ext4 /dev/sdb 
    DATA_DIR=`cat $ZK_CONFIG_FILE | grep 'dataDir=[a-z0-9/-]*' | sed 's/dataDir=//'`

    # This assumes that the dataDir value in the ZK config has a parent directory. The volume is mounted at the parent rather than the 'dataDir' value because the 
    # ext4 file system adds a 'lost+found' directory at the mount point and this confuses ZK.
    DATA_PARENT=`dirname "${DATA_DIR}"`  
    mkdir -p $DATA_PARENT
    mount /dev/sdb $DATA_PARENT
    mkdir -p $DATA_DIR
    chmod a+rwx $DATA_DIR

    # As per the dataDir
    mkfs -t ext4 /dev/sdc 
    LOG_DIR=`cat $ZK_CONFIG_FILE | grep 'dataLogDir=[a-z0-9/-]*' | sed 's/dataLogDir=//'`
    LOG_PARENT=`dirname "${LOG_DIR}"`
    mkdir -p $LOG_PARENT
    mount /dev/sdc $LOG_PARENT
    mkdir -p $LOG_DIR
    chmod a+rwx $LOG_DIR

    # Find the 'server.<number>' entry in the ZK config whose IP address matches the ENI's address, extract the <number> part and use it as the ZK server's id
    ENI_PRIVATE_IP=`aws ec2 describe-network-interfaces --filters "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --output json --query "NetworkInterfaces[0].PrivateIpAddresses[0].PrivateIpAddress" | grep -o '[0-9\.]*'`
    MATCHING_SERVER_ID=`cat $ZK_CONFIG_FILE | sed -n "s/^server.\([0-9]*\)=$ENI_PRIVATE_IP[\:0-9]*/\1/p"`
    echo $MATCHING_SERVER_ID >> $DATA_DIR/myid 

    # Use the ENI's IP address as ZK's client connection listen address
    echo >> $ZK_CONFIG_FILE
    echo "clientPortAddress=$ENI_PRIVATE_IP" >> $ZK_CONFIG_FILE

    # Get the ZK zip and install it
    ZK_INSTALL_FILE=$ZK_VERSION.tar.gz
    aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$ZK_INSTALL_FILE $ZK_INSTALL_FILE
    tar -xzf $ZK_INSTALL_FILE
    rm $ZK_INSTALL_FILE

    chown -R ec2-user:ec2-user $DATA_DIR
    chown -R ec2-user:ec2-user $LOG_DIR

}

start_zookeeper() {

    # Optional 
    echo "export JMXDISABLE=true" >> /home/ec2-user/.bash_profile
    echo "export SERVER_JVMFLAGS=-DXmx2G" >> /home/ec2-user/.bash_profile
    
    # Schedule starting the ZK server to ensure that the instance is up and all configuration has run
    # The time based scheduling could obviously be made more sophisticated should it be required.
    runuser -l ec2-user -c "echo 'nohup /home/ec2-user/$ZK_VERSION/bin/zkServer.sh start /home/ec2-user/$ZK_CONFIG_FILE &' | at now + 3 minute"

}

install_kafka() {

    # See comments in the 'install_zookeeper' function
    aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$KAFKA_CONFIG_FILE $KAFKA_CONFIG_FILE

    mkfs -t ext4 /dev/sdd
    LOG_DIR=`cat $KAFKA_CONFIG_FILE | grep 'log\.dir=[a-z0-9/-]*' | sed 's/log\.dir=//'`
    LOG_PARENT=`dirname "${LOG_DIR}"`
    mkdir -p $LOG_PARENT
    mount /dev/sdd $LOG_PARENT

    mkdir -p $LOG_DIR
    chmod a+rwx $LOG_DIR

    ENI_PRIVATE_IP=`aws ec2 describe-network-interfaces --filters "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --output json --query "NetworkInterfaces[0].PrivateIpAddresses[0].PrivateIpAddress" | grep -o '[0-9\.]*'`
    
    # Use the last octet of the ENI's IP address as the Kafka broker's id (21, 37 or 53 are used in the associated blog post)
    BROKER_ID=`echo $ENI_PRIVATE_IP | awk --field-separator \. '{ print $4 }'`
    echo broker.id=$BROKER_ID >> $KAFKA_CONFIG_FILE
    echo >> $KAFKA_CONFIG_FILE

    # Use the ENI's IP address as the Kafka listener and advertised listener values
    echo listeners=PLAINTEXT://${ENI_PRIVATE_IP}:9095 >> $KAFKA_CONFIG_FILE
    echo >> $KAFKA_CONFIG_FILE

    echo advertised.listeners=PLAINTEXT://${ENI_PRIVATE_IP}:9095 >> $KAFKA_CONFIG_FILE
    echo >> $KAFKA_CONFIG_FILE

    # Get the Kafka zip and install it
    KAFKA_INSTALL_FILE=$KAFKA_VERSION.tgz
    aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$KAFKA_INSTALL_FILE $KAFKA_INSTALL_FILE
    tar -xzf $KAFKA_INSTALL_FILE
    rm $KAFKA_INSTALL_FILE

    chown -R ec2-user:ec2-user $LOG_DIR

    PRIMARY_IP=`curl http://$AWS_METADATA_IP/latest/meta-data/local-ipv4` 
    HOST_NAME=`curl http://$AWS_METADATA_IP/latest/meta-data/local-hostname`
    SHORT_HOST_NAME=`echo $HOST_NAME | awk --field-separator \. '{ print $1 }'`

    # The default host name for an EC2 instance causes Kafka to throw an exception so create an appropriate hosts file entry which Kafka can use
    echo >> etc/hosts
    echo $PRIMARY_IP $SHORT_HOST_NAME $HOST_NAME >> /etc/hosts

}

start_kafka() {

    # Schedule starting the ZK server to ensure that the instance is up and all configuration has run.
    # The time based scheduling could obviously be made more sophisticated should it be required.
    runuser -l ec2-user -c "echo 'nohup /home/ec2-user/$KAFKA_VERSION/bin/kafka-server-start.sh /home/ec2-user/$KAFKA_CONFIG_FILE &' | at now + 4 minute"
}

attach_eni

install_zookeeper
install_kafka

chown -R ec2-user:ec2-user .

start_zookeeper
start_kafka