mesos on Vagrant

init_zookeeper.conf

description	"Zookeeper Daemon"
env JAVA_HOME=/usr/lib/jvm/jdk1.8.0_11
export JAVA_HOME

start on runlevel [2345]
stop on runlevel [!2345]

respawn

console log
chdir /home/vagrant/zookeeper-3.4.6
exec bin/zkServer.sh start-foreground

install_java.yml

---
- name: create jvm dir
  file: dest=/usr/lib/jvm owner=0 group=0 state=directory
# this is just so it doesn't bother transferring up the .tar.gz
# only to immediately skip due to the "creates"
- name: check for java
  register: java_exe_stat
  stat: path={{java_home}}/bin/java get_md5=no
- name: unpack java
  when: not java_exe_stat.stat.exists
  unarchive:
    creates: '{{java_home}}/bin/java'
    src: jdk-8u11-linux-x64.tar.gz
    dest: /usr/lib/jvm
- name: fix jdk permissions
  command: 'chown -R 0:0 {{java_home}}'
- name: set JAVA_HOME
  shell: |
    creates=/etc/profile.d/java_home.sh
    cat >/etc/profile.d/java_home.sh<<EOD
    JAVA_HOME={{java_home}}
    export JAVA_HOME
    PATH=\$JAVA_HOME/bin:\$PATH
    EOD
    # ensure there is a newline after the EOD

mesos_play.yml

---
- hosts: all
  sudo: true
  vars:
    - mesos_ver: '0.19.1'
    - mesos_home: /opt/mesos-{{mesos_ver}}
    - zoo_hosts:
        - zoo1
        - zoo2
        - zoo3
      # yup, the path only appears in _the last_ of them
      # so from the URI point of view the "host" is zoo1,zoo2,zoo3
    - mesos_master: 'zk://zoo1:2181,zoo2:2181,zoo3:2181/mesos'
    - mesos_hosts:
        - mesos1
        - mesos2
        - mesos3
  tasks:
  - name: use apt-cache if handy
    ignore_errors: true
    unarchive:
      src: var_cache_apt.tar
      dest: /
  - name: install apt dependencies
    apt:
      name: '{{item}}'
      update_cache: yes
    # this is entirely too wide, it is left over from when
    # we were building mesos on each machine
    with_items:
      - build-essential
      - openjdk-6-jdk
      - python-dev
      - python-boto
      - libcurl4-nss-dev
      - libsasl2-dev
#      - maven
#      - autoconf
#      - libtool
  - name: stat mesos
    stat: path={{mesos_home}}/bin/mesos get_md5=no
    register: mesos_stat
  - name: unpack mesos
    when: not mesos_stat.stat.exists
    unarchive:
      src: mesos-{{mesos_ver}}-bin.tar
      dest: /  # because it has the /opt/mesos already in it
  - name: patch mesos-daemon.sh
    copy:
      src: my_mesos-daemon.sh
      dest: '{{mesos_home}}/sbin/mesos-daemon.sh'
      owner: root
      mode: 0755
  - name: build mesos
    when: 1 == 0
    shell: |
      cd {{mesos_src}}
      ./bootstrap
      mkdir build
      cd build
      ../configure --prefix={{mesos_home}}
      make 2>&1|tee build.log
      make check 2>&1|tee check.log
      make install
      chown -R vagrant {{mesos_home}}
  - name: make mesos data directory
    file: path=/var/lib/mesos state=directory owner=root
  - name: make mesos log directory
    file: path=/var/log/mesos state=directory owner=root
  - name: create masters file
    copy:
      dest: '{{mesos_home}}/var/mesos/deploy/masters'
      content: |
        {{ansible_hostname}}
        # end with a newline
  - name: create mesos-master-env
    copy:
      dest: '{{mesos_home}}/var/mesos/deploy/mesos-master-env.sh'
      # it is "--master" for the slaves, but "--zk" for us
      content: |
        export MESOS_zk={{mesos_master}}
        export MESOS_log_dir=/var/log/mesos
        export MESOS_work_dir=/var/lib/mesos
        export MESOS_quorum=1
  - name: create slaves file
    copy:
      dest: '{{mesos_home}}/var/mesos/deploy/slaves'
      content: |
        {% for s in mesos_hosts 
               if s != ansible_hostname %}
        {{s}}
        {% endfor %}
        # end with a newline
  - name: create mesos-slave-env.sh
    copy:
      dest: '{{mesos_home}}/var/mesos/deploy/mesos-slave-env.sh'
      content: |
        export MESOS_master={{mesos_master}}
        export MESOS_log_dir=/var/log/mesos
        export MESOS_work_dir=/var/lib/mesos
        export MESOS_isolation=cgroups/cpu,cgroups/mem
        # end with a newline
  - name: setup known_hosts
    # regrettably this will only be the most useful at the end of the line
    shell: |
      creates=/etc/ssh/ssh_known_hosts
      grep '[0-9]$' /etc/hosts | ssh-keyscan -f - > /etc/ssh/ssh_known_hosts
  - name: ensure key based ssh
    copy:
      src: insecure_private_key
      dest: /root/.ssh/id_rsa
      owner: root
      mode: 0600
  - name: publish public ssh key
    # yes, vagant's `authorized_keys` is _usually_ just the vagrant one...
    # regardless of how it may look, `creates=` is NOT a shell assignment
    # ansible will extract it out of the command
    shell: |
      creates=/root/.ssh/id_rsa.pub
      ssh-keygen -f /root/.ssh/id_rsa -y > /root/.ssh/id_rsa.pub
      cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys

my_mesos-daemon.sh

#!/usr/bin/env bash
# set -x

prefix=/opt/mesos-0.19.1
exec_prefix=${prefix}

deploy_dir=${prefix}/var/mesos/deploy

# Increase the default number of open file descriptors.
ulimit -n 8192

PROGRAM=${1}

shift # Remove PROGRAM from the argument list (since we pass ${@} below).

test -e ${deploy_dir}/${PROGRAM}-env.sh && \
. ${deploy_dir}/${PROGRAM}-env.sh

if [ -n "${MESOS_work_dir}" ]; then
  cd ${MESOS_work_dir}
fi
echo "EXEC: ${exec_prefix}/sbin/${PROGRAM} ${@} &"
# this is probably not right long-term, but previously the script
# was swallowing error output *and* not checking to see that the
# child process actually lived past the fork; so for Vagrant situations,
# I'll take this version
${exec_prefix}/sbin/${PROGRAM} ${@} \
    >${MESOS_log_dir}/${PROGRAM}.out \
    2>${MESOS_log_dir}/${PROGRAM}.err &
echo $! > ${MESOS_work_dir}/${PROGRAM}.pid
child_pid=`cat ${MESOS_work_dir}/${PROGRAM}.pid`
sleep 5
if ! ps -p ${child_pid} >/dev/null 2>&1; then
  echo "Startup failed; pid ${child_pid} disappeared" >&2
  exit 1
fi
disown

Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby :

# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = '2'

Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  config.vm.box = 'trusty64'
  config.vm.box_url = 'http://cloud-images.ubuntu.com/vagrant/precise/current/precise-server-cloudimg-amd64-vagrant-disk1.box'
  
  zoo_host_map = {
    'zoo1' => '192.168.56.111',
    'zoo2' => '192.168.56.112',
    'zoo3' => '192.168.56.113',
  }
  mesos_host_map = {
    'mesos1' => '192.168.56.121',
    'mesos2' => '192.168.56.122',
    'mesos3' => '192.168.56.123',
  }
  
  $zoo_hosts_file = zoo_host_map.map do
    |h,ip| "#{ip} #{h}" end.join("\n")
  $zoo_update_hosts =<<ZOO
cat >>/etc/hosts<<H
#{$zoo_hosts_file}
H
ZOO
  
  $mesos_hosts_file = mesos_host_map.map do
    |h,ip| "#{ip} #{h}" end.join("\n")
  $mesos_update_hosts =<<MESOS
cat >>/etc/hosts<<H
#{$mesos_hosts_file}
#{$zoo_hosts_file}
H
MESOS
  
  zoo_host_map.keys.each do |hn|
    config.vm.define hn do |c|
      c.vm.hostname = hn
      c.vm.network :private_network, ip: zoo_host_map[c.vm.hostname]
      c.vm.provider :virtualbox do |vb|
        vb.customize ['modifyvm', :id, '--memory', '512']
      end
      c.vm.provision :shell, :inline => $zoo_update_hosts
      c.vm.provision :ansible do |a|
        a.verbose = 'vvvv'
        a.playbook = 'zoo_play.yml'
        a.extra_vars = {'hostname' => config.vm.hostname}
      end
    end
  end

  mesos_host_map.keys.each do |hn|
    config.vm.define hn do |c|
      c.vm.hostname = hn
      c.vm.network :private_network, ip: mesos_host_map[c.vm.hostname]
      c.vm.provider :virtualbox do |vb|
        vb.customize ['modifyvm', :id, '--memory', '1024']
      end
      c.vm.provision :shell, :inline => $mesos_update_hosts
      c.vm.provision :ansible do |a|
        a.verbose = 'vvvv'
        a.playbook = 'mesos_play.yml'
        a.extra_vars = {'hostname' => config.vm.hostname}
      end
    end
  end
end

zk.cfg.j2

tickTime=2000
dataDir={{zk_data}}
clientPort=2181
# how long to wait for all members to raise their hand (in ticks)
initLimit=5
# how far out of date can a member be (in ticks)
syncLimit=2
{% for it in zoos %}
server.{{loop.index}}={{it}}:2888:3888
{% endfor %}

zoo_play.yml

---
- hosts: all
  sudo: true
  vars:
    - java_home: /usr/lib/jvm/jdk1.8.0_11
    - zk_home: /home/vagrant/zookeeper-3.4.6
    - zk_data: /var/lib/zookeeper
    - zoos:
        - zoo1
        - zoo2
        - zoo3
  tasks:
    - include: install_java.yml java_home={{java_home}}
    # this is just so it doesn't bother transferring up the .tar.gz
    # only to immediately skip due to the "creates"
    - name: check for zookeeper
      stat: path={{zk_home}}/bin/zkServer.sh get_md5=no
      register: zkserver_stat
    - name: unpack zookeeper
      when: not zkserver_stat.stat.exists
      unarchive:
        src: zookeeper-3.4.6.tar.gz
        dest: /home/vagrant
        creates: '{{zk_home}}/bin/zkServer.sh'
    - name: fix zookeeper permissions
      command: 'chown -R vagrant {{zk_home}}'
    - name: create zoo data directory
      file: dest={{zk_data}} owner=vagrant state=directory
    - name: create zookeeper config
      template:
        src: zk.cfg.j2
        dest: '{{zk_home}}/conf/zoo.cfg'
        owner: vagrant
    - name: create myid file
      copy:
        dest: '{{zk_data}}/myid'
        content: '{{ansible_hostname|replace("zoo", "")}}'
        owner: vagrant
    - name: install init zookeeper.conf
      copy:
        dest: /etc/init/zookeeper.conf
        src: init_zookeeper.conf
        owner: root