Jenkins managed build script to run jobs as Milpa pods

build-via-milpa.sh

#!/bin/bash -e

# Install jq if missing.
JQ="/tmp/jq"
$JQ --version || {
    echo "Can't find jq, trying to install it"
    wget -O $JQ https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
    chmod 755 $JQ
}
echo "jq version:" $($JQ --version)

TAILING_PID=0
MILPACTL="/milpactl/milpactl --endpoints milpa:54555"
POD_NAME="pod-${BUILD_TAG}" # jenkins-${JOB_NAME}-${BUILD_NUMBER}
MANIFEST_FILE="/tmp/$POD_NAME.yml"
PKG_FILE="/tmp/$POD_NAME-pkg.tar.gz"
UNIT_NAME="executor"
IMAGE="elotl/golangbuild:latest" # Change this if you need a different base image.
NUM_CPU=${NUM_CPU:-2}
GB_RAM=${GB_RAM:-2}
GB_DISK=${GB_DISK:-8}

# Convert command line arguments into a comma delimited YAML array.
COMMAND="["
for arg in "$@"; do
    COMMAND="${COMMAND}\"${arg}\","
done
COMMAND="${COMMAND/%,/]}"

function finish {
    echo "Trying to remove pod $POD_NAME"
    $MILPACTL delete pod $POD_NAME
    if [[ $TAILING_PID -ne 0 ]]; then
        echo "Killing background task tailing logs (pid $TAILING_PID)"
        kill -9 $TAILING_PID || true
    fi
    if [[ -z "$MILPA_DEBUG" ]]; then
	rm $MANIFEST_FILE
	rm $PKG_FILE
    fi
}

function tail_logs {
    # Try to tail logs.
    set +e
    while true; do
        $MILPACTL logs -f $POD_NAME 2>&1| egrep --line-buffered -v "^Error streaming logs|^Unit $UNIT_NAME is not running|^$"
        sleep 1
    done
}

# We might have a job that's called milpa-nightly that should build 
# milpa.  Allow the user to set the app's real name
APP=${APP:-$JOB_BASE_NAME}

/bin/cat <<EOM >$MANIFEST_FILE
apiVersion: v1
kind: Pod
metadata:
  name: $POD_NAME
  annotations:
    pod.elotl.co/milpactl-volume-name: milpactl-vol
  labels:
    jenkins-build: $APP
spec:
  restartPolicy: Never
  resources:
    cpu: "${NUM_CPU}"
    memory: "${GB_RAM}Gi"
    volumeSize: "${GB_DISK}Gi"
  volumes:
  - name: milpactl-vol
    packagePath:
      path: milpactl
EOM

# Set cleanup routine.
trap finish EXIT SIGINT SIGTERM

echo "Creating pod $POD_NAME"
$MILPACTL create -f $MANIFEST_FILE

echo "Waiting for pod $POD_NAME to start running"
while true; do
    status="$($MILPACTL get pod $POD_NAME | tail -n+2 | awk '{print $5}')"
    if [[ "$status" = "Running" ]]; then
        echo "Pod $POD_NAME is now running"
        break
    fi
    sleep 5
done

# note: for parallel builds, the workspace name is something like "jobname@2"
# while the $JOB_BASE_NAME is "jobname".
WORKSPACEDIR="$(basename $WORKSPACE)"
tar --transform "s#^$WORKSPACEDIR#ROOTFS/$APP/#" -C $WORKSPACE/.. -cvzf $PKG_FILE $WORKSPACEDIR

# MILPACTL_PKG_FILE=$APP-milpactl.tar.gz
# tar --transform "s#^var/jenkins_home/milpactl/#ROOTFS/milpactl/#" -cvzf $MILPACTL_PKG_FILE /var/jenkins_home/milpactl

echo "Deploying packaged up workspace and milpactl to $POD_NAME"
$MILPACTL deploy $POD_NAME app $PKG_FILE
# $MILPACTL deploy $POD_NAME milpactl $MILPACTL_PKG_FILE

echo "Gathering Build Environment"

# we split like this since env vars can have '=' in their values
ENV_KEYS=()
ENV_VALUES=()
while read -r -d '' ENV_VAR; do
    ENV_VAR_KEY="${ENV_VAR%%=*}"
    ENV_VAR_VALUE="${ENV_VAR#*=}"
    if [[ "$ENV_VAR_KEY" == ghprb* ]]; then
        continue
    fi
    ENV_KEYS+=( "$ENV_VAR_KEY" )
    ENV_VALUES+=( "$ENV_VAR_VALUE" )
done < <(env -0)

ENV_PARAMETERS="env:\n"
for ((i=0;i<${#ENV_KEYS[@]};++i)); do
    ENV_PARAMETERS="${ENV_PARAMETERS}      - name: ${ENV_KEYS[i]}\n"
    ENV_PARAMETERS="${ENV_PARAMETERS}        value: '${ENV_VALUES[i]}'\n"
done
ENV_PARAMETERS_NEWLINE=$(printf "$ENV_PARAMETERS")

/bin/cat <<EOM >$MANIFEST_FILE
apiVersion: v1
kind: Pod
metadata:
  name: $POD_NAME
  annotations:
    pod.elotl.co/milpactl-volume-name: milpactl-vol
  labels:
    jenkins-build: $APP
spec:
  resources:
    cpu: "${NUM_CPU}"
    memory: "${GB_RAM}Gi"
  restartPolicy: Never
  units:
  - name: $UNIT_NAME
    image: $IMAGE
    command: $COMMAND
    $ENV_PARAMETERS_NEWLINE
    volumeMounts:
    - name: app
      mountPath: /go/src/github.com/elotl/$APP
    - name: milpactl-vol
      mountPath: /milpactl
    workingDir: /go/src/github.com/elotl/$APP
  volumes:
  - name: app
    packagePath:
      path: $APP
  - name: milpactl-vol
    packagePath:
      path: milpactl
EOM

# echo "Manifest file"
# echo $(cat $MANIFEST_FILE)

echo "Updating pod $POD_NAME"
$MILPACTL update -f $MANIFEST_FILE

tail_logs &
TAILING_PID=$!

echo "Waiting for pod $POD_NAME to finish running"
while true; do
    status="$($MILPACTL get pod $POD_NAME | tail -n+2 | awk '{print $5}')"
    if [[ "$status" != "Running" ]]; then
        echo "Pod $POD_NAME is not running anymore"
        break
    fi
    sleep 5
done

echo "Getting the tail end of the logs"
sleep 3

echo "Pod $POD_NAME exit status:"
$MILPACTL get pod -ojson $POD_NAME | $JQ '.status.unitStatuses[0].state'

EXITCODE=$($MILPACTL get pod -ojson $POD_NAME | $JQ '.status.unitStatuses[0].state.terminated.exitCode')
if [[ -n "$EXITCODE" ]] && [[ "$EXITCODE" -eq "$EXITCODE" ]] 2>/dev/null; then
    exit $EXITCODE
fi

exit 1