boot-ose22-pentagenous.sh

#!/bin/bash

# This script boots a nameserver; an
# arbitrary number of hosts that it configures as OpenShift brokers; an
# arbitrary number of hosts that it configures as OpenShift nodes; an
# arbitrary number of hosts that it configures as ActiveMQ brokers; and
# an arbitrary number of hosts that it configures as MongoDB replicants.

set -uo pipefail
# Don't use set -e because that interferes with the trap command.
#trap "echo Dropping to shell... ; $SHELL" ERR
set -e

: ${ASSIGN_FLOATINGIPS:=false}
: ${NUM_BROKERS:=2}
: ${NUM_NODES:=2}
: ${NUM_ACTIVEMQ:=2}
: ${NUM_MONGODB:=3}
: ${INSTALL_ROUTER:=false}

: ${INSTANCE_PREFIX:?}

: ${APPS_DOMAIN:='apps.example.com'}
: ${HOSTS_DOMAIN:='hosts.example.com'}

: ${REPOS_BASE:='http://cdn.rcm-internal.redhat.com/content/dist/rhel/server/6/6Server/x86_64'}
: ${RHEL_REPO:='http://cdn.rcm-internal.redhat.com/content/dist/rhel/server/6/6Server/x86_64/os'}

: ${POOL_ID:='8a85f9863cf496b3013da2315a27694d'}
: ${INSTALL_METHOD:=rhsm}
#INSTALL_METHOD=rhn
#INSTALL_METHOD=yum
#INSTALL_METHOD=none

: ${openshift_sh:='https://raw.githubusercontent.com/openshift/openshift-extras/enterprise-2.2/enterprise/install-scripts/generic/openshift.sh'}

: ${NOVA_FLAVOR:=a4827976-e727-4135-ba59-7d5fdb9ce4e2} #m1.small
#: ${NOVA_IMAGE:=1e45b8b3-c71a-4cc0-8d1a-b4ec2dda8ded} #RHEL6.5-qcow2-updated-20131213
: ${NOVA_IMAGE:=fd8c00c8-6c79-4317-8a38-8735e8484064} #rhel-guest-image-6-6.5-20131115.0-1.qcow2
: ${NOVA_KEY_NAME:=libra}

SCRATCHDIR="boot-ose-$INSTANCE_PREFIX"
LOGSDIR="${SCRATCHDIR}/logs"
SCRIPTSDIR="${SCRATCHDIR}/scripts"
mkdir -p "$SCRATCHDIR" "$LOGSDIR" "$SCRIPTSDIR"
exec 4>&1 > "${LOGSDIR}/boot-ose.log" 3>&1 1>&4 4>&-
BASH_XTRACEFD=3
set -x

#########################
### Utility functions ###
#########################

# $1 = iterator
# $2 = code block
# #3- = items
each() {
  iterator="$1"
  block="$2"
  shift 2
  eval "for $iterator; do $block; done"
}

# $1 = message
die() { echo "$1" >&2; exit 1; }

generate_new_password() {
  randomized="$(openssl rand -base64 20)"
  printf '%s' "${randomized//[![:alnum:]]}"
}


######################
### Main functions ###
######################

# $1 = instance name
boot() {
  [[ $# -eq 1 ]] || die "boot(): need 1 argument, got $#"

  local name="${INSTANCE_PREFIX}-$1"
  local bootscript="${SCRIPTSDIR}/cloud-init-${name}.sh"

  exec >>"${LOGSDIR}/${name}.log" 2>&1
  generate_cloud_init_script "$1" > "$bootscript"
  local -a cmd=( nova boot "--flavor=${NOVA_FLAVOR}" "--image=${NOVA_IMAGE}" "--key_name=${NOVA_KEY_NAME}" "--security-groups=$(generate_security_groups "$1")" "--user-data=$bootscript" "$name" --poll )
  local retries=10
  while :
  do
    "${cmd[@]}"
    addr="$(get_external_address "$1")"
    if [[ -n "$addr" ]]
    then
      echo "Booting instance $1 (${addr}) succeeded."
      break
    elif (( retries > 0 ))
    then
      echo "Booting instance $1 failed.  Will retry up to $((retries--)) more times."
      echo -n 'Cleaning up failed instance...'
      nova delete "$name"
      echo 'Done.'
    else
      die "Booting instance $1 failed."
    fi
  done
  [[ "$ASSIGN_FLOATINGIPS" = true ]] && nova add-floating-ip "$name" "$addr"
}

# $1 = instance name
generate_security_groups()
{
  case "${1%%[[:digit:]]*}" in
    (activemq)  printf '%s' 'default,activemq' ;;
    (broker)    printf '%s' 'default,activemq,datastore,nameserver,broker' ;;
    (datastore) printf '%s' 'default,datastore' ;;
    (node)      printf '%s' 'default,node' ;;
    (ns)        printf '%s' 'default,nameserver' ;;
    (router)    printf '%s' 'default,broker' ;;
# XXX Use the above for OS1 Internal and the below for OS1 Public.  Need to come
# up with a better approach.
#    (activemq)  printf '%s' 'default,openshift-broker' ;;
#    (broker)    printf '%s' 'default,openshift-broker' ;;
#    (datastore) printf '%s' 'default,openshift-broker' ;;
#    (node)      printf '%s' 'default,openshift-node' ;;
#    (ns)        printf '%s' 'default,openshift-broker' ;;
    (*)         die "generate_security_groups(): could not discern instance type from name: $1" ;;
  esac
}

# $1 = instance name
generate_cloud_init_script()
{
  cat <<EOF
#!/bin/bash

# Unblock root ssh logins.
sed -i -e 's/,command=.*\\( ssh-rsa\\)/\1/' /root/.ssh/authorized_keys

# The RHEL product cert is missing on the official RHEL 6.5 cloud VM.
mkdir -p /etc/pki/product
curl 'http://file.rdu.redhat.com/~bleanhar/69.pem' -o /etc/pki/product/69.pem

# Download and run the installation script.
curl '${openshift_sh}' -o /root/openshift.sh
#export CONF_OSE_EXTRA_REPO_BASE='http://buildvm-devops.usersys.redhat.com/puddle/build/OpenShiftEnterpriseErrata/2.2/2014-10-28.1/'
sh -x /root/openshift.sh $(generate_openshift_sh_arguments "$1") |& tee -a /root/openshift.sh.log
EOF
}

# $1 = instance name
#
# The following variables must be set:
# ACTIVEMQ_REPLICANTS
# ACTIVEMQ_AMQ_PASSWORD
# ACTIVEMQ_MCOLLECTIVE_PASSWORD
# ROUTING_PLUGIN_PASS
# MONGODB_BROKER_PASSWORD
# MONGODB_KEY
# DATASTORE_REPLICANTS
# APPS_DOMAIN
# HOSTS_DOMAIN
# INSTALL_ARGUMENTS
# NAMESERVER_DNSSEC_KEY
generate_openshift_sh_arguments()
{
  case "${1%%[[:digit:]]*}" in
    (activemq)   printf '%s' "$INSTALL_ARGUMENTS $ROUTER_ARGUMENTS $DOMAINS $NAMED_IP_ADDR install_components=activemq  activemq_replicants=${ACTIVEMQ_REPLICANTS} activemq_hostname=$1.${HOSTS_DOMAIN} activemq_amq_user_password=${ACTIVEMQ_AMQ_PASSWORD} routing_plugin_pass=${ROUTING_PLUGIN_PASS} mcollective_password=${ACTIVEMQ_MCOLLECTIVE_PASSWORD}" ;;
    (broker)     printf '%s' "$INSTALL_ARGUMENTS $ROUTER_ARGUMENTS $DOMAINS $NAMED_IP_ADDR install_components=broker    activemq_replicants=${ACTIVEMQ_REPLICANTS} datastore_replicants=${DATASTORE_REPLICANTS} bind_key=${NAMESERVER_DNSSEC_KEY} broker_hostname=$1.${HOSTS_DOMAIN} routing_plugin_pass=${ROUTING_PLUGIN_PASS} mcollective_password=${ACTIVEMQ_MCOLLECTIVE_PASSWORD} mongodb_broker_password=${MONGODB_BROKER_PASSWORD} mongodb_key=${MONGODB_KEY} openshift_password1=changeme" ;;
    (datastore)  printf '%s' "$INSTALL_ARGUMENTS $ROUTER_ARGUMENTS $DOMAINS $NAMED_IP_ADDR install_components=datastore datastore_replicants=${DATASTORE_REPLICANTS} datastore_hostname=$1.${HOSTS_DOMAIN} mongodb_broker_password=${MONGODB_BROKER_PASSWORD} mongodb_key=${MONGODB_KEY}" ;;
    (node)       printf '%s' "$INSTALL_ARGUMENTS $ROUTER_ARGUMENTS $DOMAINS $NAMED_IP_ADDR install_components=node      cartridges=standard,-jboss,screen,bind-utils,mlocate activemq_replicants=${ACTIVEMQ_REPLICANTS} node_hostname=$1.${HOSTS_DOMAIN} broker_hostname=broker01.${HOSTS_DOMAIN} mcollective_password=${ACTIVEMQ_MCOLLECTIVE_PASSWORD}" ;;
    (ns)         printf '%s' "$INSTALL_ARGUMENTS $ROUTER_ARGUMENTS $DOMAINS                install_components=named     bind_key=${NAMESERVER_DNSSEC_KEY}" ;;
    (router)     printf '%s' "$INSTALL_ARGUMENTS $ROUTER_ARGUMENTS $DOMAINS $NAMED_IP_ADDR install_components=router    activemq_replicants=${ACTIVEMQ_REPLICANTS} routing_plugin_pass=${ROUTING_PLUGIN_PASS}" ;;
    (*)          die "generate_openshift_sh_arguments(): could not discern instance type from name: $1" ;;
  esac
}

# $1 = instance name
# $2- = command
#
# Runs the specified command on the specified instance.
on_instance() {
  instance="$1"
  shift
  ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "root@$(get_external_address "$instance")" "$@"
}

# $1 = instance name
# $2- = extra arguments to openshift.sh
#
# Runs openshift.sh on the specified instance with the appropriate arguments
# for the host as well as any additional arguments that may be specified.
openshift_sh() {
  local instance="$1"
  local name="${INSTANCE_PREFIX}-$1"
  shift
  on_instance "$instance" sh /root/openshift.sh $(generate_openshift_sh_arguments "$instance") "$@" >>"${LOGSDIR}/${name}.log" 2>&1
}

generate_dnssec_key()
{
  KEYDIR="${SCRATCHDIR}/keys"
  mkdir -p "$KEYDIR"
  grep Key: "$KEYDIR/$(dnssec-keygen -a HMAC-SHA256 -b 256 -n USER -r /dev/urandom -K "$KEYDIR" hosts.sample.com).private" | cut -d ' ' -f 2
}

# $1 = instance name
wait_for_boot() {
  [[ $# -eq 1 ]] || die "wait_for_boot(): need 1 argument, got $#"

  echo -n "Waiting for ${1} to start..."
  while :
  do
    on_instance "$1" -q /bin/true && break || :
    echo -n '.'
    sleep 2
  done
  echo 'Ready.'
}

# $1 = instance name
# $2 = service name
wait_for_service() {
  # Use uniq here to avoid printing "named: unrecognized service" over and over.
  on_instance "$1" "while ! service \"$2\" status; do sleep 5; done" |& uniq
}

# $1 = instance name
declare -A _get_external_address
get_external_address() {
  [[ $# -eq 1 ]] || die "get_external_address(): need 1 argument, got $#"

  echo -n ${_get_external_address[$1]:=$(nova show "${INSTANCE_PREFIX}-$1" | sed -n -e '/.* network .* [0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+, \([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\).*/ {s//\1/;p;q;}')}
}

# $1 = instance name
declare -A _get_internal_address
get_internal_address() {
  [[ $# -eq 1 ]] || die "get_internal_address(): need 1 argument, got $#"

  echo -n ${_get_internal_address[$1]:=$(nova show "${INSTANCE_PREFIX}-$1" | sed -n -e '/.* network .* [0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+, \([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\).*/ {s//\1/;p;q;}')}
}

####################
### Main section ###
####################

echo "Using prefix \"${INSTANCE_PREFIX}.\""

echo "Generating keys and passwords..."
ACTIVEMQ_AMQ_PASSWORD="$(generate_new_password)"
ACTIVEMQ_MCOLLECTIVE_PASSWORD="$(generate_new_password)"
ROUTING_PLUGIN_PASS="$(generate_new_password)"
MONGODB_BROKER_PASSWORD="$(generate_new_password)"
MONGODB_KEY="$(generate_new_password)"
NAMESERVER_DNSSEC_KEY="$(generate_dnssec_key)"
printf 'ActiveMQ amq user password: %s\n' "$ACTIVEMQ_AMQ_PASSWORD"
printf 'ActiveMQ mcollective user password: %s\n' "$ACTIVEMQ_MCOLLECTIVE_PASSWORD"
printf 'Activemq routinginfo user password: %s\n' "$ROUTING_PLUGIN_PASS"
printf 'MongoDB openshift user password: %s\n' "$MONGODB_BROKER_PASSWORD"
printf 'MongoDB key: %s\n' "$MONGODB_KEY"
printf 'DNSSEC key: %s\n' "$NAMESERVER_DNSSEC_KEY"

case "$INSTALL_METHOD" in
  (rhsm) INSTALL_ARGUMENTS="install_method=rhsm sm_reg_pool=${POOL_ID} rhn_user=${RHN_USER} rhn_pass=${RHN_PASS}" ;;
  (rhn)  INSTALL_ARGUMENTS="install_method=rhn rhn_user=${RHN_USER} rhn_pass=${RHN_PASS}" ;;
  (yum)  INSTALL_ARGUMENTS="install_method=yum repos_base=${REPOS_BASE} rhel_repo=${RHEL_REPO}" ;;
esac

ROUTER_ARGUMENTS=
if [[ "$INSTALL_ROUTER" = true ]]
then ROUTER_ARGUMENTS="enable_ha=true router=nginx router_hostname=router.${HOSTS_DOMAIN}"
fi

DOMAINS="domain=${APPS_DOMAIN} hosts_domain=${HOSTS_DOMAIN}"

declare -a broker_instances=( $(eval echo broker{01..$NUM_BROKERS}) )
declare -a node_instances=( $(eval echo node{01..$NUM_NODES}) )
declare -a activemq_instances=( $(eval echo activemq{01..$NUM_ACTIVEMQ}) )
declare -a mongodb_instances=( $(eval echo datastore{01..$NUM_MONGODB}) )
declare router_instance=
[[ "$INSTALL_ROUTER" = true ]] && router_instance=router
# Note that $instances will not include ns1.
declare -a instances=( ${broker_instances[@]} ${node_instances[@]} ${activemq_instances[@]} ${mongodb_instances[@]} $router_instance )

# This is kind of a hack...
# If ASSIGN_FLOATINGIPS=true, then get a list of floatingips and seed
# _get_external_address with these addresses.  The boot() function will actually
# assign them if ASSIGN_FLOATINGIPS=true.
#floatingips=( $(nova floating-ip-list | awk '/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/ {print $2}') )
#peek_floatingip() { idx=$((${#floatingips[@]}-1)) ; echo -n "${floatingips[$idx]}" ; }
#pop_floatingip()  { idx=$((${#floatingips[@]}-1)) ; unset floatingips[$idx] ; }
#each instance '_get_external_address[$instance]=$(peek_floatingip); pop_floatingip' ns1 ${instances[@]}
get_floatingip() {
  nova floating-ip-create |
    sed -n -e '/.* \([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\) .*/ s//\1/p'
}
# XXX Enable the following line only for OS1 Public and not for OS1 Internal.
#each instance '_get_external_address[$instance]=$(get_floatingip)' ns1 ${instances[@]}

ACTIVEMQ_REPLICANTS="$(echo ${activemq_instances[@]} | sed -e "s/ \\|\$/.${HOSTS_DOMAIN} /g;s/ \$//;y/ /,/")"
DATASTORE_REPLICANTS="$(echo ${mongodb_instances[@]} | sed -e "s/ \\|\$/.${HOSTS_DOMAIN} /g;s/ \$//;y/ /,/")"

# We need the nameserver's IP address to configure everything else, so
# boot the ns1 first in the background.
echo 'Booting nameserver instance...'
boot ns1 &
echo 'Waiting for nameserver instance to boot...'
wait
NAMED_IP_ADDR="named_ip_addr=$(get_internal_address ns1)"

# Boot everything but ns1 in parallel (sleep avoids rate-limiting).
echo 'Booting other instances...'
each instance 'echo "Booting ${instance}..." && boot "$instance" & sleep 2' ${instances[@]}

# Wait for everything to boot so everything will have IP addresses
# assigned.
echo 'Waiting for instances to boot...'
wait

echo 'Press Enter to continue.'; read x
wait_for_boot ns1
wait_for_service ns1 named
echo 'Adding DNS records for instances...'
named_entries="$(each instance 'printf ,%s:%s $instance.${HOSTS_DOMAIN} $(get_internal_address "$instance")' ${instances[@]})"
on_instance ns1 'service named stop'
openshift_sh ns1 'actions=configure_hosts_dns' "named_entries=${named_entries:1}"
on_instance ns1 'service named start'

echo 'Waiting for MongoDB instances to start...'
each instance 'wait_for_boot "$instance"' ${mongodb_instances[@]}
each instance 'openshift_sh "$instance" actions=wait_for_mongod' ${mongodb_instances[@]}
if [[ $NUM_MONGODB -gt 1 ]]
then
  echo 'Initiating MongoDB replication...'
  openshift_sh datastore01 'actions=configure_datastore_add_replicants'
fi

echo 'Waiting for ActiveMQ, OpenShift broker, and OpenShift node instances to start...'
each instance 'wait_for_boot "$instance"' ${activemq_instances[@]} ${broker_instances[@]} ${node_instances[@]}
echo 'Waiting 30 seconds for hosts to settle...'
sleep 30

echo 'Waiting for ruby193-mcollective to start on node instances...'
each instance 'wait_for_service "$instance" ruby193-mcollective' ${node_instances[@]}
echo 'Waiting for openshift-broker to start on broker01...'
wait_for_service broker01 openshift-broker
echo 'Performing post-installation steps on broker01...'
each instance 'openshift_sh "$instance" actions=post_deploy' "${broker_instances[0]}" "${node_instances[@]}"
if [[ "$INSTALL_ROUTER" = true ]]
then
  echo 'Copying TLS cert and key from node01 to router...'
  scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
    "root@$(get_external_address node01):/etc/pki/tls/certs/localhost.crt" \
    /tmp/node.example.com.crt
  scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
    /tmp/node.example.com.crt \
    "root@$(get_external_address router):/etc/pki/tls/certs/node.example.com.crt"
  scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
    "root@$(get_external_address node01):/etc/pki/tls/private/localhost.key" \
    /tmp/node.example.com.key
  scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
    /tmp/node.example.com.key \
    "root@$(get_external_address router):/etc/pki/tls/private/node.example.com.key"
  rm -f /tmp/node.example.com.crt /tmp/node.example.com.key
  on_instance router 'service nginx16-nginx start'
fi

echo 'Done.'