Strigo's Development Environment Provisioning Script

strigo-deploy-dev-env.sh

#!/usr/bin/env bash

###
# The script uses CFN to provision a Target Group in the app-dev ALB, and creates
# an instance behind that target group, after which it  attaches two host-header
# based rules to the HTTP/HTTPS listeners which target the newly created target group.
# After the stack is up, it installs the relevant Strigo app, ssc and job-worker builds
# on it for the relevant branch.
#
# NOTE: AWS_DEFAULT_REGION, AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set prior to running this.
###

set -euo pipefail

declare -r APP_DEV_LISTENER_ARN='arn:aws:elasticloadbalancing:...'

function log() {
  message="$1"

  echo -e "\e[32m## ${message}\e[0m" >&2
}

function error_exit() {
  #######################################
  # Print a title in red
  # TODO: echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $@" >&2
  # Arguments:
  #   The message to be printed
  #######################################
  local message="$1"

  echo "\e[91m${message}" >&2
  exit 1
}

function install_python_dependency() {
  #######################################
  # Install dependencies required for stack provisioning
  #######################################
  local package="$1"

  if ! is_python_package_installed ${package}; then
    log "Installing ${package}..."
    sudo pip install ${package} --upgrade
  fi
}

function is_python_package_installed() {
  local package_name="$1"

  log "Verifying ${package_name} is installed..."
  if pip freeze | grep ${package_name} >/dev/null; then
    return 0
  fi
  return 1
}

function get_alb_listener_rule_priority() {
  #######################################
  # Get the required priority (1, 2, etc..) for the rule.
  # Each rule must be given a priority when it is created.
  #######################################
  log "Retrieving listener rule priority..."
  local priority=$(aws elbv2 describe-rules --region ${AWS_DEFAULT_REGION} --listener-arn ${APP_DEV_LISTENER_ARN} | jq '[.Rules[].Priority][:-1]' | jq '[.[] | tonumber] | max + 1')
  log "Priority for ${APP_DEV_LISTENER_ARN} is: ${priority}"

  echo ${priority}
}

function is_stack_exists() {
  #######################################
  # Check whether the stack `stack_name` exists.
  # Note that this will not check if the stack exists and was created successfully.
  #######################################
  local stack_name="$1"

  if aws cloudformation describe-stacks | jq ".Stacks[].StackName | contains(\"${stack_name}\")" | grep "true" >/dev/null; then
    log "Stack already exists"
    return 0
  fi
  log "Stack does not exist"
  return 1
}

function create_stack() {
  #######################################
  # Idempotently create a stack
  #######################################
  local branch="$1"
  local priority="$2"

  log "Creating stack for branch: ${branch}..."
  aws cloudformation create-stack --region ${AWS_DEFAULT_REGION} --stack-name "${branch}" --template-body "$(cat dev/cfn.yaml)" --parameters ParameterKey=Branch,ParameterValue="${branch}" ParameterKey=LoadBalancerPriority,ParameterValue=$priority
  log "Waiting for creation to complete..."
  aws cloudformation wait stack-create-complete --region ${AWS_DEFAULT_REGION} --stack-name "${branch}"
  log "Done"
}

function delete_stack() {
  #######################################
  # Idempotently create a stack
  #######################################
  local branch="$1"

  log "Deleting stack for branch: ${branch}..."
  aws cloudformation delete-stack --region ${AWS_DEFAULT_REGION} --stack-name "${branch}"
  log "Waiting for deletion to complete..."
  aws cloudformation wait stack-delete-complete --region ${AWS_DEFAULT_REGION} --stack-name "${branch}"
  log "Done"
}

function delete_resources() {
  #######################################
  # Idempotently create a stack
  #######################################
  local branch="$1"

  # Making sure we don't accidentally pass an empty branch name in which case
  # aws s3 rm will delete the entire bucket's contents.
  if ! is_var_set ${branch}; then
    return
  fi
  log "Deleting resources for branch: ${branch}..."
  aws s3 rm --region ${AWS_DEFAULT_REGION} s3://strigo-deploy-dev/${branch} --recursive
  log "Done"
}

function get_instance_id() {
  local branch="$1"

  log "Retrieving instance id for stack: ${branch}..."
  local instance_id=$(aws cloudformation describe-stack-resources --region ${AWS_DEFAULT_REGION} --stack-name "${branch}" | jq -r .StackResources[0].PhysicalResourceId)
  log "Instance ID is: ${instance_id}"

  echo ${instance_id}
}

function get_instance_ip() {
  local intsance_id="$1"

  log "Retrieving instance ip for instance id: ${instance_id}..."
  local instance_ip=$(aws ec2 describe-instances --region ${AWS_DEFAULT_REGION} --instance-id ${instance_id} | jq -r .Reservations[].Instances[].PrivateIpAddress)
  log "Instance IP is: ${instance_ip}"

  echo ${instance_ip}
}

function is_dir() {
  local dir="$1"

  if [[ -d ${dir} ]]; then
    return 0
  fi
  return 1
}

function is_var_set() {
  local var="$1"

  if [[ -z ${var} || -R ${var} ]]; then
    return 1
  fi
  return 0
}

function edit_hosts_file() {
  #######################################
  # Place instance ip in hosts file
  # Used by ansible to provision the machine
  #######################################
  local instance_ip="$1"
  local hosts_file_path="$2"

  log "Editing hosts file: ${hosts_file_path}..."
  sed -i "s/{{INSTANCE_IP}}/${instance_ip}/g" ${hosts_file_path}
  log "Hosts file is:"
  echo "$(cat ${hosts_file_path})" >&2
}

function clone_ansible_source() {
  #######################################
  # Clone or pull strigo-ops repo.
  #######################################
  local branch="$1"
  local repo_name="strigo-ops"
  local repo_slug="strigo/${repo_name}"
  local clone=false

  if ! is_dir ${repo_name}; then
    clone=true
    log "Cloning ${repo_slug}..."
    git clone [email protected]:${repo_slug}.git
  fi

  pushd ${repo_name} >/dev/null

  if is_var_set ${branch}; then
    log "Checking out ${repo_slug}/${branch}..."
    git checkout ${branch}
  fi

  if ! [ ${clone} = true ]; then
    log "Pulling branch..."
    git pull
  fi
  popd >/dev/null
}

function set_ssh_key() {
  #######################################
  # Write and add strigo's ssh key to the agent
  # for ansible to be able to provision the instance
  #######################################
  local key="$1"
  local key_file="${2:-private_key}"

  log "Writing instance ssh key to: ${key_file}..."
  echo -e "$key" > ${key_file}
  chmod 600 ${key_file}
  log "Adding instance ssh key agent..."
  ssh-add ${key_file}
  ssh-add -l
}

function ansible() {
  #######################################
  # Deploy strigo-app, strigo-job-worker and strigo-ssc
  # On the provisioned instance (via ansible)
  #######################################
  local working_dir="$1"
  local branch="$2"
  local hosts_file="$3"
  local strigo_app_version="$4"
  local stack_created_now="$5"

  local strigo_ssc_version="${6:-0.0.5}"
  local strigo_job_worker_version="${7:-0.0.3}"

  local vars="hosts=strigo-instance env=dev branch=\"${branch}\" aws_access_key=\"${AWS_ACCESS_KEY_ID}\" aws_secret_key=\"${AWS_SECRET_ACCESS_KEY}\""
  local args="-i ${hosts_file} -vv"

  if ${stack_created_now}; then
    local strigo_app_tags='--skip-tags datadog'
    local ansible_tags='--skip-tags common,install_python,datadog'
  else
    local strigo_app_tags='--tags deploy'
    local ansible_tags='--tags deploy'
  fi

  pushd ${working_dir} >/dev/null

  log "Running ansible..."
  log "args: ${args}"
  log "vars: ${vars}"
  log "Hosts file: ${hosts_file}"
  log "Branch: ${branch}"
  log "strigo-app version: ${strigo_app_version}"

  ansible-playbook ${args} strigo-app.yml -e "${vars} deb_dir=${branch}/strigo-app" -e "{'strigo_app': {'version':'$strigo_app_version'}}" ${strigo_app_tags}
  ansible-playbook ${args} strigo-job-worker.yml -e "${vars}" -e "{'strigo_job_worker': {'version':'$strigo_job_worker_version'}}" ${ansible_tags}
  ansible-playbook ${args} strigo-ssc.yml -e "${vars}" -e "{'strigo_ssc': {'version':'$strigo_ssc_version'}}" ${ansible_tags}
  ansible-playbook ${args} strigo-mongo-dev.yml -e "${vars}" ${ansible_tags}

  popd >/dev/null
}

function assert_required_vars_set() {
  local required_unset=false

  # TODO: Nicefy
  if ! is_var_set ${AWS_ACCESS_KEY_ID}; then
    echo "AWS_ACCESS_KEY_ID must be set!"
    required_unset=true
  fi
  if ! is_var_set ${AWS_SECRET_ACCESS_KEY}; then
    echo "AWS_SECRET_ACCESS_KEY must be set!"
    required_unset=true
  fi
  if ! is_var_set ${AWS_DEFAULT_REGION}; then
    echo "AWS_DEFAULT_REGION must be set!"
    required_unset=true
  fi
  if ${required_unset}; then
    error_exit "Some required environment variables are not set!"
  fi
}


function main() {
  # TODO: Add normal CLI
  local branch="$1"
  local instance_ssh_key="$2"
  local strigo_app_version="$3"
  local commit_message="$4"

  local strigo_ops_branch="${5:-master}"
  local hosts_file="${6:-dev/hosts}"

  assert_required_vars_set

  install_python_dependency "awscli"
  local stack_created_now=false
  local stack_exists=true
  if ! is_stack_exists "${branch}"; then
    stack_exists=false
  fi

  # Only want to create the stack if commit message is `make_env` and it doesn't exist.
  if [[ "${commit_message}" == 'make_env' ]] && ! ${stack_exists}; then
    log "Stack creation required."
    local priority=$(get_alb_listener_rule_priority)
    create_stack "${branch}" "${priority}"
    stack_created_now=true
    stack_exists=true
  elif [[ "${commit_message}" == 'destroy_env' ]] && ${stack_exists}; then
    log "Stack deletion required."
    delete_stack "${branch}"
    delete_resources "${branch}"
    exit 0
  fi

  # Only want to try and deploy if the stack exists.
  if ${stack_exists}; then
    install_python_dependency "ansible==2.3.0.0"
    log "Deployment required."
    local instance_id=$(get_instance_id "${branch}")
    local instance_ip=$(get_instance_ip "${instance_id}")
    edit_hosts_file "${instance_ip}" "${hosts_file}"
    clone_ansible_source "${strigo_ops_branch}"
    set_ssh_key "${instance_ssh_key}"
    ansible "strigo-ops/ansible" "${branch}" "../../${hosts_file}" "${strigo_app_version}" "${stack_created_now}"
  else
    log "Skipping deploy."
  fi
}

main "$@"