agent-setup.sh

# On a new ubuntu 14.04 box, as root, with the following variables

# export BUILDBOX_AGENT_NAME="buildbox-agent-N"
# export BUILDBOX_AGENT_ACCESS_TOKEN="<your token here>"
# export BUILDBOX_AGENT_SSH_PRIVATE_KEY="<an ssh private key>"
# export BUILDBOX_AGENT_SSH_PUBLIC_KEY="<an ssh public key>"

# Make tmp totally in memory
echo "none /tmp tmpfs size=4g 0 0" >> /etc/fstab
mount /tmp

# We need certain hosts
echo "127.0.0.1 3docean.dev activeden.dev audiojungle.dev codecanyon.dev envatomarketplaces.dev graphicriver.dev marketplace.envato.dev photodune.dev themeforest.dev videohive.dev" >> /etc/hosts

# And out hostname
if ! grep -q "$BUILDBOX_AGENT_NAME" /etc/hosts; then
  echo "127.0.0.1 $BUILDBOX_AGENT_NAME" >> /etc/hosts
fi

if [ "$(cat /etc/hostname)" != "$BUILDBOX_AGENT_NAME" ]; then
  echo "$BUILDBOX_AGENT_NAME" > /etc/hostname
fi

if [ "$(hostname)" != "$BUILDBOX_AGENT_NAME" ]; then
  hostname "$BUILDBOX_AGENT_NAME"
fi

# Install basics

aptitude -q -y update
aptitude -q -y upgrade
aptitude -q -y install git curl zip unzip zlib1g-dev build-essential libssl-dev libreadline-dev libyaml-dev libcurl4-openssl-dev python-software-properties libffi-dev libncurses5-dev libicu-dev

# Make sure we're on Melbourne time, and stay current

echo "Australia/Melbourne" > /etc/timezone
dpkg-reconfigure --frontend noninteractive tzdata

apt-get -q -y install ntp

# Install mysql

# http://dba.stackexchange.com/questions/59317/install-mariadb-10-on-ubuntu-without-prompt-and-no-root-password#answer-60192
cat <<-EOF | debconf-set-selections
  mysql-server-5.5 mysql-server/root_password password temp
  mysql-server-5.5 mysql-server/root_password seen true
  mysql-server-5.5 mysql-server/root_password_again password temp
  mysql-server-5.5 mysql-server/root_password_again seen true
EOF
aptitude -q -y install mysql-server libmysqlclient-dev
mysql -uroot -ptemp -e "SET PASSWORD = PASSWORD('')"

# Make mysql storage ram-based
stop mysql
mv /var/lib/mysql /var/lib/mysql.init
mkdir /var/lib/mysql
echo "none /var/lib/mysql tmpfs size=2g,mode=0755,uid=mysql,gid=mysql 0 0" >> /etc/fstab
mount /var/lib/mysql
https://gist.githubusercontent.com/sj26/d27de45269e2051bb8ca/raw/init-mysql.conf -O /etc/init/mysql.conf
start mysql

# Install redis

aptitude -q -y install redis-server

# We need more databases
sed -i -e "s/databases [0-9]\+/databases 256/" /etc/redis/redis.conf

sudo /etc/init.d/redis-server restart

# Install nodejs for execjs

aptitude -q -y install nodejs npm

# ubuntu is dumb
ln -s $(which nodejs) /usr/local/bin/node

# Install phantomjs for selenium / jasmine

aptitude -q -y install phantomjs

# Install firefox for selenium / jasmine

aptitude -q -y install xvfb firefox

# Install chromium for selenium / jasmine

aptitude -q -y install xvfb chromium-browser chromium-chromedriver

# Chromium webdriver doesn't understand where to look for chrome libraries
echo "/usr/lib/chromium-browser/libs" > /etc/ld.so.conf.d/chromedriver.conf
ldconfig

# and we want to make sure things can find it
ln -s /usr/lib/chromium-browser/chromedriver /usr/local/bin/chromedriver

# Install sphinx

aptitude -q -y install sphinxsearch

# Install Elasticsearch

aptitude -q -y install openjdk-7-jre-headless

pushd /tmp
wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.1.2.deb
dpkg -i elasticsearch-1.1.2.deb
update-rc.d elasticsearch defaults 95 10
/etc/init.d/elasticsearch start
popd

pushd /usr/share/elasticsearch
bin/plugin -install elasticsearch/elasticsearch-lang-javascript/2.1.0
/etc/init.d/elasticsearch restart
popd

# Make elasticsearch storage ram-based
/etc/init.d/elasticsearch stop
mv /var/lib/elasticsearch /var/lib/elasticsearch.clean
mkdir /var/lib/elasticsearch
echo "none /var/lib/elasticsearch tmpfs size=2g,mode=0755,uid=elasticsearch,gid=elasticsearch 0 0" >> /etc/fstab
mount /var/lib/elasticsearch
cp -rp /var/lib/elasticsearch.clean/* /var/lib/elasticsearch
rm /etc/init.d/elasticsearch
https://gist.githubusercontent.com/sj26/d27de45269e2051bb8ca/raw/init-elasticsearch.conf -O /etc/init/elasticsearch.conf
start elasticsearch

# Install stuff for gems

aptitude -y install libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev libexiv2-dev imagemagick libmagickwand-dev

# Install rbenv (system level)

git clone https://github.com/sstephenson/rbenv /opt/rbenv
mkdir -p /opt/rbenv/{cache,plugins}

git clone https://github.com/sstephenson/ruby-build /opt/rbenv/plugins/ruby-build
git clone https://github.com/sstephenson/rbenv-default-gems /opt/rbenv/plugins/rbenv-default-gems
git clone https://github.com/sstephenson/rbenv-vars /opt/rbenv/plugins/rbenv-vars
git clone https://github.com/tpope/rbenv-communal-gems /opt/rbenv/plugins/rbenv-communal-gems

cat <<-"EOF" >/opt/rbenv/default-gems
  gem-open
  bundler
EOF

cat <<-"EOF" >/etc/profile.d/rbenv.sh
  # Prefer a user rbenv over a system wide install
  if [ -s "${HOME}/.rbenv/bin" ]; then
    rbenv_root="${HOME}/.rbenv"
  elif [ -s "/opt/rbenv" ]; then
    rbenv_root="/opt/rbenv"
    export RBENV_ROOT="$rbenv_root"
  fi
  
  # Initialize rbenv if found
  if [ -n "$rbenv_root" ]; then
    export PATH="${rbenv_root}/bin:$PATH"
    eval "$(rbenv init -)"
  fi
EOF

source /etc/profile.d/rbenv.sh

# Install a recent global ruby
# (envato rubby needs a base ruby to install)

rbenv install 2.1.2
rbenv global 2.1.2

# Install the one true envato rubby

pushd /tmp
wget https://gist.github.com/charliesome/241081793396463e706e/raw/f7e26da65ae1990174f5695a81f0c41c3aa02784/1.9.3%2Benvato4
rbenv install 1.9.3+envato4
popd

# JavaScript tests need testem

npm install testem -g

# Setup selenium

useradd --system --home /opt/selenium --shell /bin/bash selenium
install -d -o selenium -g selenium /opt/selenium
wget http://selenium-release.storage.googleapis.com/2.42/selenium-server-standalone-2.42.2.jar -O /opt/selenium/selenium-server-standalone-2.42.2.jar
wget https://gist.github.com/sj26/d27de45269e2051bb8ca/raw/node-config.json -O /opt/selenium/node-config.json
wget https://gist.github.com/sj26/d27de45269e2051bb8ca/raw/selenium-node.conf -O /etc/init/selenium-node.conf

start selenium-node

# Setup buildbox account

useradd --system --home /opt/buildbox --shell /bin/bash buildbox
install -d -o buildbox -g buildbox /opt/buildbox
sudo -u buildbox mkdir ~buildbox/.ssh
# sudo -u buildbox ssh-keygen -q -t rsa -b 2048 -C "buildbox@$(hostname)" -f ~buildbox/.ssh/id_rsa -N ""
echo "$BUILDBOX_AGENT_SSH_PRIVATE_KEY" > ~buildbox/.ssh/id_rsa
echo "$BUILDBOX_AGENT_SSH_PUBLIC_KEY" > ~buildbox/.ssh/id_rsa.pub
chown buildbox:buildbox ~buildbox/.ssh/id_rsa ~buildbox/.ssh/id_rsa.pub
chmod 0600 ~buildbox/.ssh/id_rsa
sudo -i -u buildbox bash -c "ssh-keyscan github.com > ~/.ssh/known_hosts"

# Make builds operate totally in memory
echo "none /opt/buildbox/builds tmpfs size=8g,mode=0755,uid=buildbox,gid=buildbox 0 0" >> /etc/fstab
mount /opt/buildbox/builds

# Install buildbox agent

VERSION=0.2-beta.10 DESTINATION=/opt/buildbox bash -c "`curl -sL https://raw.github.com/buildboxhq/buildbox-agent/master/install.sh`"

wget https://gist.githubusercontent.com/sj26/d27de45269e2051bb8ca/raw/bootstrap.sh -O /opt/buildbox/bootstrap.sh
wget https://gist.githubusercontent.com/sj26/d27de45269e2051bb8ca/raw/buildbox-agent.conf -O /etc/init/buildbox-agent.conf

echo "$BUILDBOX_AGENT_ACCESS_TOKEN" > /etc/buildbox_agent_access_token

start buildbox-agent

agent-user-data.sh

#!/bin/bash

# Then you can make an AMI and use this script as userdata,
# substituting a different hostname and access token:

BUILDBOX_AGENT_NAME="buildbox-agent-N"
BUILDBOX_AGENT_ACCESS_TOKEN="TOKEN"

if ! grep -q "$BUILDBOX_AGENT_NAME" /etc/hosts; then
  echo "127.0.0.1 $BUILDBOX_AGENT_NAME" >> /etc/hosts
fi

if [ "$(cat /etc/hostname)" != "$BUILDBOX_AGENT_NAME" ]; then
  echo "$BUILDBOX_AGENT_NAME" > /etc/hostname
fi

if [ "$(hostname)" != "$BUILDBOX_AGENT_NAME" ]; then
  hostname "$BUILDBOX_AGENT_NAME"
fi

echo "$BUILDBOX_AGENT_ACCESS_TOKEN" > /etc/buildbox_agent_access_token

bootstrap.sh

#!/bin/bash

# TODO: These env vars probably need to live somewhere else

# Use local selenium hub
export SELENIUM_REMOTE="http://localhost:4444/wd/hub"

# Makes ruby not GC much
export RUBY_GC_HEAP_INIT_SLOTS=1000000
export RUBY_HEAP_SLOTS_INCREMENT=1000000
export RUBY_HEAP_SLOTS_GROWTH_FACTOR=1
export RUBY_GC_MALLOC_LIMIT=1000000000
export RUBY_HEAP_FREE_MIN=500000

BUILDBOX_PROMPT="\033[90m$\033[0m"

function buildbox-exit-if-failed {
  if [ $1 -ne 0 ]
  then
    exit $1
  fi
}

function buildbox-run {
  echo -e "$BUILDBOX_PROMPT $1"
  eval $1
  buildbox-exit-if-failed $?
}

echo '--- Setup environment'

# This will return the location of this file. We assume that the buildbox-artifact
# tool is in the same folder. You can of course customize the locations
# and edit this file.
BUILDBOX_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# Create the build directory
BUILDBOX_BUILD_DIR="builds/$BUILDBOX_PROJECT_SLUG"

buildbox-run "mkdir -p $BUILDBOX_BUILD_DIR"
buildbox-run "cd $BUILDBOX_BUILD_DIR"

# Do we need to do a git checkout?
if [ ! -d ".git" ]
then
  buildbox-run "git clone --config 'remote.origin.fetch=+refs/builds/*:refs/builds/*' '$BUILDBOX_REPO' . -qv"
fi

# Default empty branch names
if [ "$BUILDBOX_BRANCH" == "" ]
then
  BUILDBOX_BRANCH="master"
fi

buildbox-run "git clean -fdq"
buildbox-run "git fetch -q"
buildbox-run "git reset --hard origin/$BUILDBOX_BRANCH"
buildbox-run "git checkout -qf \"$BUILDBOX_COMMIT\""

echo "--- Running $BUILDBOX_SCRIPT_PATH"

if [ "$BUILDBOX_SCRIPT_PATH" == "" ]
then
  echo "ERROR: No script path has been set for this project. Please go to \"Project Settings\" and add the path to your build script"
  exit 1
else
  ."/$BUILDBOX_SCRIPT_PATH"
  EXIT_STATUS=$?
fi

if [ "$BUILDBOX_ARTIFACT_PATHS" != "" ]
then
  # Make sure the buildbox-artifact binary is in the right spot.
  if [ ! -f $BUILDBOX_DIR/buildbox-artifact ]
  then
    echo >&2 "ERROR: buildbox-artifact could not be found in $BUILDBOX_DIR"
    exit 1
  fi

  # If you want to upload artifacts to your own server, uncomment the lines below
  # and replace the AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY with keys to your
  # own bucket.
  # export AWS_SECRET_ACCESS_KEY=yyy
  # export AWS_ACCESS_KEY_ID=xxx
  # buildbox-artifact upload "$BUILDBOX_ARTIFACT_PATHS" "s3://name-of-your-s3-bucket/$BUILDBOX_JOB_ID" --url $BUILDBOX_AGENT_API_URL

  # By default we silence the buildbox-artifact build output. However, if you'd like to see
  # it in your logs, remove the: > /dev/null 2>&1 from the end of the line.
  echo "--- Uploading artifacts"
  $BUILDBOX_DIR/buildbox-artifact upload "$BUILDBOX_ARTIFACT_PATHS" --url $BUILDBOX_AGENT_API_URL
  #buildbox-exit-if-failed $?
fi

exit $EXIT_STATUS

buildbox-agent.conf

description "buildbox agent"

start on stopped cloud-final # Start after cloudinit has run, so we pick up the right access token.
stop on runlevel [!2345]

respawn

script
  su - buildbox -c "xvfb-run --server-num=100 --server-args '-screen 0 1024x768x24' /opt/buildbox/buildbox-agent start --access-token $(cat /etc/buildbox_agent_access_token) --bootstrap-script /opt/buildbox/bootstrap.sh"
end script

init-elasticsearch.conf

description "Elasticsearch Server"

start on (MOUNTED=/var/lib/elasticsearch) and runlevel [2345]
stop on starting rc RUNLEVEL=[016]

respawn
respawn limit 2 5

setuid elasticsearch
setgid elasticsearch

umask 007

env HOME=/usr/share/elasticsearch

env ES_HOME=/usr/share/elasticsearch
env ES_MIN_MEM=256m
env ES_MAX_MEM=2g

pre-start script
  POTENTIAL_JAVA_HOME="/usr/lib/jvm/java-7-oracle /usr/lib/jvm/java-7-openjdk /usr/lib/jvm/java-7-openjdk-amd64/ /usr/lib/jvm/java-7-openjdk-armhf /usr/lib/jvm/java-7-openjdk-i386/ /usr/lib/jvm/java-6-sun /usr/lib/jvm/java-6-openjdk /usr/lib/jvm/java-6-openjdk-amd64 /usr/lib/jvm/java-6-openjdk-armhf /usr/lib/jvm/java-6-openjdk-i386 /usr/lib/jvm/default-java"

  # Look for the right JVM to use
  for jdir in $POTENTIAL_JAVA_HOME; do
    if [ -r "$jdir/bin/java" -a -z "${JAVA_HOME}" ]; then
      export JAVA_HOME="$jdir"
    fi
  done

  if [ -f /etc/default/elasticsearch ]; then
    . /etc/default/elasticsearch
  fi

  if [ -x "$JAVA_HOME/bin/java" ]; then
    JAVA="$JAVA_HOME/bin/java"
  else
    JAVA=`which java`
  fi

  if [ ! -x "$JAVA" ]; then
    echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
    exit 1
  fi

  if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then
    log_failure_msg "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set"
    exit 1
  fi

  # Prepare environment
  mkdir -p "$LOG_DIR" "$DATA_DIR" "$WORK_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR" "$WORK_DIR"

  if [ -n "$MAX_OPEN_FILES" ]; then
    ulimit -n $MAX_OPEN_FILES
  fi

  if [ -n "$MAX_LOCKED_MEMORY" ]; then
    ulimit -l $MAX_LOCKED_MEMORY
  fi

  if [ -n "$MAX_MAP_COUNT" ]; then
    sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
  fi
end script

exec /usr/share/elasticsearch/bin/elasticsearch -f -Des.path.conf=/etc/elasticsearch -Des.path.home=/usr/share/elasticsearch -Des.path.logs=/var/log/elasticsearch -Des.path.data=/var/lib/elasticsearch -Des.path.work=/tmp/elasticsearch

init-mysql.conf

# MySQL Service

description     "MySQL Server"
author          "Mario Limonciello <[email protected]>"

start on (MOUNTED=/var/lib/mysql) and runlevel [2345]
stop on starting rc RUNLEVEL=[016]

respawn
respawn limit 2 5

env HOME=/etc/mysql
umask 007

# The default of 5 seconds is too low for mysql which needs to flush buffers
kill timeout 300

pre-start script
    ## Fetch a particular option from mysql's invocation.
    # Usage: void mysqld_get_param option
    mysqld_get_param() {
      /usr/sbin/mysqld --print-defaults \
        | tr " " "\n" \
        | grep -- "--$1" \
        | tail -n 1 \
        | cut -d= -f2
    }

    # priority can be overriden and "-s" adds output to stderr
    ERR_LOGGER="logger -p daemon.err -t /etc/init/mysql.conf -i"

    #Sanity checks
    [ -r $HOME/my.cnf ]
    [ -d /var/run/mysqld ] || install -m 755 -o mysql -g root -d /var/run/mysqld
    /lib/init/apparmor-profile-load usr.sbin.mysqld

    # check for diskspace shortage
    datadir=`mysqld_get_param datadir`
    BLOCKSIZE=`LC_ALL=C df --portability $datadir/. | tail -n 1 | awk '{print $4}'`
    if [ $BLOCKSIZE -le 4096 ] ; then
      echo "$0: ERROR: The partition with $datadir is too full!" >&2
      echo "ERROR: The partition with $datadir is too full!" | $ERR_LOGGER
      exit 1
    fi

    # copy template into place
    rm -rf /var/lib/mysql/*
    cp -rp /var/lib/mysql.init/* /var/lib/mysql/
end script

exec /usr/sbin/mysqld

post-start script
   for i in `seq 1 30` ; do
        /usr/bin/mysqladmin --defaults-file="${HOME}"/debian.cnf ping && {
            exec "${HOME}"/debian-start
            # should not reach this line
            exit 2
        }
        statusnow=`status`
        if echo $statusnow | grep -q 'stop/' ; then
            exit 0
        elif echo $statusnow | grep -q 'respawn/' ; then
            exit 1
        fi
        sleep 1
    done
    exit 1
end script

node-config.json

{
  "capabilities": [
    {
      "browserName": "firefox",
      "maxInstances": 32,
      "seleniumProtocol": "WebDriver"
    },
    {
      "browserName": "chrome",
      "maxInstances": 32,
      "seleniumProtocol": "WebDriver"
    },
    {
      "browserName": "phantomjs",
      "maxInstances": 32,
      "seleniumProtocol": "WebDriver"
    }
  ],
  "configuration": {
    "proxy": "org.openqa.grid.selenium.proxy.DefaultRemoteProxy",
    "maxSession": 64,
    "port": 5555,
    "host": "127.0.0.1",
    "register": true,
    "registerCycle": 5000,
    "hubPort": 4444,
    "hubHost": "127.0.0.1"
  }
}

selenium-node.conf

description "selenium node"

start on (starting buildbox-agent)
stop on (stopped buildbox-agent)

respawn

script
  su - selenium -c "xvfb-run --server-num=99 --server-args '-screen 0 1024x768x24' java -jar /opt/selenium/selenium-server-standalone-2.42.2.jar -nodeConfig /opt/selenium/node-config.json"
end script