Buildbox agent setup (elastic)

agent-setup.sh

# On a new ubuntu 14.04 box, as root, with the following variables

# export BUILDBOX_AGENT_TOKEN="<your token here>"
# export BUILDBOX_AGENT_SSH_PRIVATE_KEY="<ssh private key>"
# export BUILDBOX_AGENT_SSH_PUBLIC_KEY="<ssh public key>"

# Make tmp totally in memory
echo "none /tmp tmpfs size=8g 0 0" >> /etc/fstab
mount /tmp

# We need certain hosts
echo "127.0.0.1 3docean.dev activeden.dev audiojungle.dev codecanyon.dev envatomarketplaces.dev graphicriver.dev marketplace.envato.dev photodune.dev themeforest.dev videohive.dev" >> /etc/hosts

# Install basics

aptitude -q -y update
aptitude -q -y upgrade
aptitude -q -y install git curl zip unzip zlib1g-dev build-essential libssl-dev libreadline-dev libyaml-dev libcurl4-openssl-dev python-software-properties libffi-dev libncurses5-dev libicu-dev

# Make sure we're on Melbourne time, and stay current

echo "Australia/Melbourne" > /etc/timezone
dpkg-reconfigure --frontend noninteractive tzdata

apt-get -q -y install ntp

# Make sure we can use git new-workdir
chmod +x /usr/share/doc/git/contrib/workdir/git-new-workdir
ln -s /usr/share/doc/git/contrib/workdir/git-new-workdir /usr/local/bin/git-new-workdir

# Install mysql

# http://dba.stackexchange.com/questions/59317/install-mariadb-10-on-ubuntu-without-prompt-and-no-root-password#answer-60192
cat <<-EOF | debconf-set-selections
  mysql-server-5.5 mysql-server/root_password password temp
  mysql-server-5.5 mysql-server/root_password seen true
  mysql-server-5.5 mysql-server/root_password_again password temp
  mysql-server-5.5 mysql-server/root_password_again seen true
EOF
aptitude -q -y install mysql-server libmysqlclient-dev
mysql -uroot -ptemp -e "SET PASSWORD = PASSWORD('')"

# Make mysql storage ram-based
# The replacement upstart script copies the template database into place on the newly mounted tmpfs
stop mysql
mv /var/lib/mysql /var/lib/mysql.clean
mkdir /var/lib/mysql
echo "none /var/lib/mysql tmpfs size=2g,mode=0755,uid=mysql,gid=mysql 0 0" >> /etc/fstab
mount /var/lib/mysql
wget https://gist.githubusercontent.com/sj26/d27de45269e2051bb8ca/raw/init-mysql.conf -O /etc/init/mysql.conf
start mysql

# Install redis

aptitude -q -y install redis-server

# We need more databases
sed -i -e "s/databases [0-9]\+/databases 256/" /etc/redis/redis.conf

sudo /etc/init.d/redis-server restart

# Install nodejs for execjs

aptitude -q -y install nodejs npm

# ubuntu is dumb
ln -s $(which nodejs) /usr/local/bin/node

# Install phantomjs for selenium / jasmine

# Ubuntu ships 1.9.0, we want 1.9.7, so ppa time
add-apt-repository -y ppa:igb/ppa-trusty
aptitude -q -y update
aptitude -q -y install phantomjs

# Install firefox for selenium / jasmine

aptitude -q -y install xvfb firefox

# Install chromium for selenium / jasmine

aptitude -q -y install xvfb chromium-browser chromium-chromedriver

# Chromium webdriver doesn't understand where to look for chrome libraries
echo "/usr/lib/chromium-browser/libs" > /etc/ld.so.conf.d/chromedriver.conf
ldconfig

# and we want to make sure things can find it
ln -s /usr/lib/chromium-browser/chromedriver /usr/local/bin/chromedriver

# Install ffmpeg for screen capture videos
aptitude -q -y install ffmpeg

# Install sphinx

aptitude -q -y install sphinxsearch

# Install Elasticsearch

aptitude -q -y install openjdk-7-jre-headless

pushd /tmp
wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.1.1.deb
dpkg -i elasticsearch-1.1.1.deb
update-rc.d elasticsearch defaults 95 10
/etc/init.d/elasticsearch start
popd

pushd /usr/share/elasticsearch
bin/plugin -install elasticsearch/elasticsearch-lang-javascript/2.1.0
/etc/init.d/elasticsearch restart
popd

# Make elasticsearch storage ram-based
/etc/init.d/elasticsearch stop
mv /var/lib/elasticsearch /var/lib/elasticsearch.clean
mkdir /var/lib/elasticsearch
echo "none /var/lib/elasticsearch tmpfs size=2g,mode=0755,uid=elasticsearch,gid=elasticsearch 0 0" >> /etc/fstab
mount /var/lib/elasticsearch
cp -rp /var/lib/elasticsearch.clean/* /var/lib/elasticsearch
rm /etc/init.d/elasticsearch
wget https://gist.githubusercontent.com/sj26/d27de45269e2051bb8ca/raw/init-elasticsearch.conf -O /etc/init/elasticsearch.conf
start elasticsearch

# For sqlite3 gem, *shudder*

aptitude -y install libsqlite3-dev sqlite3

# For Nokogiri, should some poor sod be using an old version without a vendored libxml/xslt

aptitude -y install libxml2-dev libxslt1-dev

# For image processing gems

aptitude -y install libexiv2-dev imagemagick libmagickwand-dev

# For geoip gems

aptitude -y install geoip-bin libgeoip-dev

# (we install libmaxminddb here as well)

# For capybara-webkit, *shudder*

aptitude -y install libqt4-dev qt4-qmake

# Install rbenv (system level)

git clone https://github.com/sstephenson/rbenv /opt/rbenv
mkdir -p /opt/rbenv/{cache,plugins}

git clone https://github.com/sstephenson/ruby-build /opt/rbenv/plugins/ruby-build
git clone https://github.com/sstephenson/rbenv-default-gems /opt/rbenv/plugins/rbenv-default-gems
git clone https://github.com/sstephenson/rbenv-vars /opt/rbenv/plugins/rbenv-vars
git clone https://github.com/tpope/rbenv-communal-gems /opt/rbenv/plugins/rbenv-communal-gems

cat <<-"EOF" >/opt/rbenv/default-gems
  gem-open
  bundler
EOF

cat <<-"EOF" >/etc/profile.d/rbenv.sh
  # Prefer a user rbenv over a system wide install
  if [ -s "${HOME}/.rbenv/bin" ]; then
    rbenv_root="${HOME}/.rbenv"
  elif [ -s "/opt/rbenv" ]; then
    rbenv_root="/opt/rbenv"
    export RBENV_ROOT="$rbenv_root"
  fi
  
  # Initialize rbenv if found
  if [ -n "$rbenv_root" ]; then
    export PATH="${rbenv_root}/bin:$PATH"
    eval "$(rbenv init -)"
  fi
EOF

source /etc/profile.d/rbenv.sh

# Install a recent global ruby
# (envato rubby needs a base ruby to install)

rbenv install 2.1.3
rbenv global 2.1.3

rbenv install 2.1.2

# Install jruby for search

rbenv install jruby-1.7.5

# Doesn't behave well with communized gems
rbenv sequester jruby-1.7.5

# And now we need to specifically install bundler
RBENV_VERSION=jruby-1.7.5 gem install bundler

# Install the one true envato rubby

pushd /tmp
wget https://gist.github.com/charliesome/241081793396463e706e/raw/f7e26da65ae1990174f5695a81f0c41c3aa02784/1.9.3%2Benvato4
rbenv install 1.9.3+envato4
popd

# Setup selenium

useradd --system --home /opt/selenium --shell /bin/bash selenium
install -d -o selenium -g selenium /opt/selenium
wget http://selenium-release.storage.googleapis.com/2.42/selenium-server-standalone-2.42.2.jar -O /opt/selenium/selenium-server-standalone-2.42.2.jar
wget https://gist.github.com/sj26/d27de45269e2051bb8ca/raw/node-config.json -O /opt/selenium/node-config.json
wget https://gist.github.com/sj26/d27de45269e2051bb8ca/raw/selenium-node.conf -O /etc/init/selenium-node.conf

# selenium-node is particularly noisy so rotate logs frequently and only keep a little bit
wget https://gist.github.com/sj26/d27de45269e2051bb8ca/raw/logrotate-selenium-node -O /etc/logrotate.d/selenium-node
cp /etc/cron.daily/logrotate /etc/cron.hourly

start selenium-node

# Setup buildbox account

useradd --system --home /opt/buildbox --shell /bin/bash buildbox
install -d -o buildbox -g buildbox /opt/buildbox
sudo -u buildbox mkdir ~buildbox/.ssh
# sudo -u buildbox ssh-keygen -q -t rsa -b 2048 -C "buildbox@$(hostname)" -f ~buildbox/.ssh/id_rsa -N ""
echo "$BUILDBOX_AGENT_SSH_PRIVATE_KEY" > ~buildbox/.ssh/id_rsa
echo "$BUILDBOX_AGENT_SSH_PUBLIC_KEY" > ~buildbox/.ssh/id_rsa.pub
chown buildbox:buildbox ~buildbox/.ssh/id_rsa ~buildbox/.ssh/id_rsa.pub
chmod 0600 ~buildbox/.ssh/id_rsa
sudo -i -u buildbox bash -c "ssh-keyscan github.com > ~/.ssh/known_hosts"

# Make builds operate totally in memory
mkdir /opt/buildbox/builds
echo "none /opt/buildbox/builds tmpfs size=8g,mode=0755,uid=buildbox,gid=buildbox 0 0" >> /etc/fstab
mount /opt/buildbox/builds

# Install buildbox agent

VERSION=1.0-beta.2 DESTINATION=/opt/buildbox bash -c "`curl -sL https://raw.github.com/buildboxhq/buildbox-agent/master/install.sh`"

wget https://gist.githubusercontent.com/sj26/d27de45269e2051bb8ca/raw/init-buildbox-agent.conf -O /etc/init/buildbox-agent.conf

wget https://gist.githubusercontent.com/sj26/d27de45269e2051bb8ca/raw/bootstrap.sh -O /opt/buildbox/bootstrap.sh
chmod +x /opt/buildbox/bootstrap.sh

wget https://gist.githubusercontent.com/sj26/d27de45269e2051bb8ca/raw/buildbox-agent-start.sh -O /opt/buildbox/bin/buildbox-agent-start.sh
chmod +x /opt/buildbox/bin/buildbox-agent-start.sh

echo "$BUILDBOX_AGENT_TOKEN" > /etc/buildbox_agent_token

start buildbox-agent

bootstrap.sh

#!/bin/bash

# TODO: These env vars probably need to live somewhere else

# Makes ruby not GC much
export RUBY_GC_HEAP_INIT_SLOTS=1000000
export RUBY_HEAP_SLOTS_INCREMENT=1000000
export RUBY_HEAP_SLOTS_GROWTH_FACTOR=1
export RUBY_GC_MALLOC_LIMIT=1000000000
export RUBY_HEAP_FREE_MIN=500000

BUILDBOX_PROMPT="\033[90m$\033[0m"

function buildbox-exit-if-failed {
  if [ $1 -ne 0 ]; then
    exit $1
  fi
}

function buildbox-run {
  echo -e "$BUILDBOX_PROMPT $1"
  eval $1
  buildbox-exit-if-failed $?
}

echo '--- Setup environment'

# This will return the location of this file. We assume that the buildbox-artifact
# tool is in the same folder. You can of course customize the locations
# and edit this file.
BUILDBOX_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# Use a common git dir
BUILDBOX_REPO_MD5="$(echo "$BUILDBOX_REPO" | md5sum | awk '{ print $1 }')"
BUILDBOX_GIT_DIR="$BUILDBOX_DIR/builds/.gits/$BUILDBOX_REPO_MD5"

# Make sure it's up to date
if [ ! -d "$BUILDBOX_GIT_DIR"  ]; then
  buildbox-run "mkdir -p $BUILDBOX_GIT_DIR"
  buildbox-run "git clone --no-checkout --config 'remote.origin.fetch=+refs/builds/*:refs/builds/*' '$BUILDBOX_REPO' '$BUILDBOX_GIT_DIR'"
else
  buildbox-run "cd '$BUILDBOX_GIT_DIR'"
  buildbox-run "git fetch"
fi

# Default empty branch names
if [ "$BUILDBOX_BRANCH" == "" ]; then
  BUILDBOX_BRANCH="master"
fi

# Make sure we have a working tree for the actual build on the correct branch
BUILDBOX_BUILD_DIR="$BUILDBOX_DIR/builds/$BUILDBOX_PROJECT_SLUG"

if [ ! -d "$BUILDBOX_BUILD_DIR" ]; then
  buildbox-run "git new-workdir '$BUILDBOX_GIT_DIR' '$BUILDBOX_BUILD_DIR' '$BUILDBOX_BRANCH'"
  buildbox-run "cd '$BUILDBOX_BUILD_DIR'"
else
  buildbox-run "cd '$BUILDBOX_BUILD_DIR'"
  buildbox-run "git checkout -f '$BUILDBOX_BRANCH'"
fi

# Reset to the correct commit
buildbox-run "git reset --hard '$BUILDBOX_COMMIT'"

echo "--- Running $BUILDBOX_SCRIPT_PATH"

if [ "$BUILDBOX_SCRIPT_PATH" == "" ]; then
  echo "ERROR: No script path has been set for this project. Please go to \"Project Settings\" and add the path to your build script"
  exit 1
fi

."/$BUILDBOX_SCRIPT_PATH"
EXIT_STATUS=$?

if [ "$BUILDBOX_ARTIFACT_PATHS" != "" ]; then
  # Make sure the buildbox-artifact binary is in the right spot.
  if [ ! -f $BUILDBOX_DIR/bin/buildbox-agent ]; then
    echo >&2 "ERROR: buildbox-agent could not be found in $BUILDBOX_DIR"
    exit 1
  fi

  echo "--- Uploading artifacts"
  $BUILDBOX_DIR/bin/buildbox-agent artifact upload "$BUILDBOX_ARTIFACT_PATHS"

  # If you want to upload artifacts to your own server:
  # export AWS_SECRET_ACCESS_KEY=yyy
  # export AWS_ACCESS_KEY_ID=xxx
  # buildbox artifact upload "$BUILDBOX_ARTIFACT_PATHS" "s3://name-of-your-s3-bucket/$BUILDBOX_JOB_ID"

  # We don't want to fail on artifact upload failure
  #buildbox-exit-if-failed $?
fi

exit $EXIT_STATUS

buildbox-agent-start.sh

#!/bin/bash

# Buildbox agent configuration
export BUILDBOX_AGENT_NAME="$(ec2metadata --public-hostname)"
export BUILDBOX_AGENT_TOKEN="$(cat /etc/buildbox_agent_token)"
export BUILDBOX_BOOTSTRAP_SCRIPT_PATH="/opt/buildbox/bootstrap.sh"

export BUILDBOX_AGENT_META_DATA="role=test"
for version in $(rbenv versions --bare); do
  export BUILDBOX_AGENT_META_DATA="$BUILDBOX_AGENT_META_DATA,ruby=$version"
done
export BUILDBOX_AGENT_META_DATA="$BUILDBOX_AGENT_META_DATA,elasticsearch=1.1.1"
export BUILDBOX_AGENT_META_DATA="$BUILDBOX_AGENT_META_DATA,mysql=5.5"
export BUILDBOX_AGENT_META_DATA="$BUILDBOX_AGENT_META_DATA,redis=true"
export BUILDBOX_AGENT_META_DATA="$BUILDBOX_AGENT_META_DATA,selenium-remote=true"
export BUILDBOX_AGENT_META_DATA="$BUILDBOX_AGENT_META_DATA,firefox=true"
export BUILDBOX_AGENT_META_DATA="$BUILDBOX_AGENT_META_DATA,chromium=true"
export BUILDBOX_AGENT_META_DATA="$BUILDBOX_AGENT_META_DATA,phantomjs=true"
export BUILDBOX_AGENT_META_DATA="$BUILDBOX_AGENT_META_DATA,nodejs=true"

# Publish configuration for services available to builds
# Inside the build steps we'll use SELENIUM_REMOTE=true to trigger using it
export SELENIUM_REMOTE_URL="http://localhost:4444/wd/hub"

exec xvfb-run --server-num=10 --server-args "-screen 0 1024x768x24" /opt/buildbox/bin/buildbox-agent start

init-buildbox-agent.conf

description "buildbox agent"

# Give cloud-init a chance to (re)populate /etc/buildbox_agent_access_token
start on runlevel [2345] and started mysql and started elasticsearch and started selenium-node and stopped cloud-final
stop on runlevel [!2345] or stopping mysql or stopping elasticsearch or stopping selenium-node

respawn

exec su - buildbox -c /opt/buildbox/bin/buildbox-agent-start.sh

init-elasticsearch.conf

description "Elasticsearch Server"

start on runlevel [2345]
stop on starting rc RUNLEVEL=[016]

respawn
respawn limit 2 5

setuid elasticsearch
setgid elasticsearch

umask 007

env HOME=/usr/share/elasticsearch

env ES_HOME=/usr/share/elasticsearch
env ES_MIN_MEM=256m
env ES_MAX_MEM=2g

pre-start script
  POTENTIAL_JAVA_HOME="/usr/lib/jvm/java-7-oracle /usr/lib/jvm/java-7-openjdk /usr/lib/jvm/java-7-openjdk-amd64/ /usr/lib/jvm/java-7-openjdk-armhf /usr/lib/jvm/java-7-openjdk-i386/ /usr/lib/jvm/java-6-sun /usr/lib/jvm/java-6-openjdk /usr/lib/jvm/java-6-openjdk-amd64 /usr/lib/jvm/java-6-openjdk-armhf /usr/lib/jvm/java-6-openjdk-i386 /usr/lib/jvm/default-java"

  # Look for the right JVM to use
  for jdir in $POTENTIAL_JAVA_HOME; do
    if [ -r "$jdir/bin/java" -a -z "${JAVA_HOME}" ]; then
      export JAVA_HOME="$jdir"
    fi
  done

  if [ -f /etc/default/elasticsearch ]; then
    . /etc/default/elasticsearch
  fi

  if [ -x "$JAVA_HOME/bin/java" ]; then
    JAVA="$JAVA_HOME/bin/java"
  else
    JAVA=`which java`
  fi

  if [ ! -x "$JAVA" ]; then
    echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
    exit 1
  fi

  if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then
    log_failure_msg "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set"
    exit 1
  fi

  # Prepare environment
  mkdir -p "$LOG_DIR" "$DATA_DIR" "$WORK_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR" "$WORK_DIR"

  if [ -n "$MAX_OPEN_FILES" ]; then
    ulimit -n $MAX_OPEN_FILES
  fi

  if [ -n "$MAX_LOCKED_MEMORY" ]; then
    ulimit -l $MAX_LOCKED_MEMORY
  fi

  if [ -n "$MAX_MAP_COUNT" ]; then
    sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
  fi
end script

exec /usr/share/elasticsearch/bin/elasticsearch -f -Des.path.conf=/etc/elasticsearch -Des.path.home=/usr/share/elasticsearch -Des.path.logs=/var/log/elasticsearch -Des.path.data=/var/lib/elasticsearch -Des.path.work=/tmp/elasticsearch

init-mysql.conf

# MySQL Service

description     "MySQL Server"
author          "Mario Limonciello <[email protected]>"

start on runlevel [2345]
stop on starting rc RUNLEVEL=[016]

respawn
respawn limit 2 5

env HOME=/etc/mysql
umask 007

# The default of 5 seconds is too low for mysql which needs to flush buffers
kill timeout 300

pre-start script
    ## Fetch a particular option from mysql's invocation.
    # Usage: void mysqld_get_param option
    mysqld_get_param() {
      /usr/sbin/mysqld --print-defaults \
        | tr " " "\n" \
        | grep -- "--$1" \
        | tail -n 1 \
        | cut -d= -f2
    }

    # priority can be overriden and "-s" adds output to stderr
    ERR_LOGGER="logger -p daemon.err -t /etc/init/mysql.conf -i"

    #Sanity checks
    [ -r $HOME/my.cnf ]
    [ -d /var/run/mysqld ] || install -m 755 -o mysql -g root -d /var/run/mysqld
    /lib/init/apparmor-profile-load usr.sbin.mysqld

    # check for diskspace shortage
    datadir=`mysqld_get_param datadir`
    BLOCKSIZE=`LC_ALL=C df --portability $datadir/. | tail -n 1 | awk '{print $4}'`
    if [ $BLOCKSIZE -le 4096 ] ; then
      echo "$0: ERROR: The partition with $datadir is too full!" >&2
      echo "ERROR: The partition with $datadir is too full!" | $ERR_LOGGER
      exit 1
    fi

    # copy template into place
    rm -rf /var/lib/mysql/*
    cp -rp /var/lib/mysql.clean/* /var/lib/mysql/
end script

exec /usr/sbin/mysqld

post-start script
   for i in `seq 1 30` ; do
        /usr/bin/mysqladmin --defaults-file="${HOME}"/debian.cnf ping && {
            exec "${HOME}"/debian-start
            # should not reach this line
            exit 2
        }
        statusnow=`status`
        if echo $statusnow | grep -q 'stop/' ; then
            exit 0
        elif echo $statusnow | grep -q 'respawn/' ; then
            exit 1
        fi
        sleep 1
    done
    exit 1
end script

logrotate-selenium-node

# Selenium node logs are particularly verbose, so only keep a
# little bit and compress immediately. -- @sj26
/var/log/upstart/selenium-node.log {
  hourly
  missingok
  rotate 1
  compress
  nodelaycompress
  notifempty
  nocreate
}

node-config.json

{
  "capabilities": [
    {
      "browserName": "firefox",
      "maxInstances": 32,
      "seleniumProtocol": "WebDriver"
    },
    {
      "browserName": "chrome",
      "maxInstances": 32,
      "seleniumProtocol": "WebDriver"
    },
    {
      "browserName": "phantomjs",
      "maxInstances": 32,
      "seleniumProtocol": "WebDriver"
    }
  ],
  "configuration": {
    "proxy": "org.openqa.grid.selenium.proxy.DefaultRemoteProxy",
    "maxSession": 64,
    "port": 5555,
    "host": "127.0.0.1",
    "register": true,
    "registerCycle": 5000,
    "hubPort": 4444,
    "hubHost": "127.0.0.1",
    "browserTimeout": 60000
  }
}

selenium-node.conf

description "selenium node"

start on runlevel [2345]
stop on runlevel [!2345]

respawn

script
  su - selenium -c "xvfb-run --server-num=11 --server-args '-screen 0 1024x768x24' java -jar /opt/selenium/selenium-server-standalone-2.42.2.jar -nodeConfig /opt/selenium/node-config.json"
end script

user-data-to-change-agent-token.sh

#!/bin/bash

echo "$token" > /etc/buildbox_agent_token

user-data-to-prevent-agent-running.sh

#!/bin/bash

# To boot up an AMI and tweak the agent before re-imaging you don't want to run
# the buildbox-agent on boot and run builds, so we can stop the agent before it
# starts in the cloud-init process by doing the following:

stop buildbox-agent