aurora client binding hooks madness (unfinished proof of concept)

binding_helper.aurora.py

import subprocess

from apache.aurora.client.binding_helper import (BindingHelper,
                                                 CachingBindingHelper)
from apache.aurora.client.config import GlobalHookRegistry
from pystachio.matcher import Any, Matcher
from twitter.common import log


class NixHelper(CachingBindingHelper):
  @property
  def name(self):
    return 'Nix'

  @property
  def matcher(self):
    return Matcher('Nix.*')[Any]

  def uncached_bind(self, config, match, env, binding_dict):
    package = match[1]
    ref = Ref.from_address('%s[%s]' % match)

    nix_attrib = nix_packages[package]
    store_path = self._nix_realise(nix_attrib)

    binding = {ref: store_path}
    config.bind(binding)
    return binding

  def _nix_realise(self, attribute):
    log.info('Assembling: %s', attribute)
    store_path = subprocess.check_output(
        ['nix-build', '-A', attribute, '--no-out-link']).strip()

    log.debug('Store path: %s', store_path)
    return store_path


BindingHelper.register(NixHelper())

nix.aurora.py

# Use nix without a persistent user profile, which means no post-run cleanup
simple_nixproc = Process(
  name = '{{binary}}',
  cmdline = unindent(
    r"""
    export NIX_REMOTE=daemon
    if ! which nix-store; then
      export PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:$PATH
    fi
    nix-store --add-root ".gc/{{name}}" --indirect -r "{{store_path}}"
    export PATH=$PWD/.gc/{{name}}/bin:$PWD/.gc/{{name}}/sbin:$PATH

    exec {{binary}} {{args}}
    """))

prometheus.aurora.py

# There are several terrible ideas in this file.
# Running prometheus this way *will* result in data loss. Use for development only.

include('lib/nix.aurora')
include('lib/binding_helper.aurora')

# close enough
go_options = java_options

nix_packages = {
  'aurora_exporter': 'prometheus-aurora-exporter',
  'configs': 'monitoring.prometheus.configs',
  'gen_file_sd': 'monitoring.prometheus.gen_file_sd',
  'mesos_exporter': 'prometheus-mesos-exporter',
  'mksynapse': 'mksynapse',
  'prometheus': 'prometheus',
}

proc_gentargets = Process(
  name = 'gentargets',
  ephemeral = True,
  daemon = True,
  min_duration = 60,
  cmdline = unindent(
    r"""
    set -x
    export PATH=/nix/var/nix/profiles/default/bin:$PATH
    export NIX_REMOTE=daemon
    nix-store -r --add-root ".gc/{{name}}" --indirect "{{Nix[gen_file_sd]}}"
    export PATH="$PWD/.gc/{{name}}/bin:$PATH"

    mkdir -p target_groups
    while true; do
      # Atomic file replacement is important here:
      gen_file_sd us-east-1 > target_groups/._node.yml
      mv target_groups/._node.yml target_groups/node.yml
      sleep 60
    done
    """))

proc_s3checkpoint = Process(
  name = 's3checkpoint',
  ephemeral = True,
  daemon = True,
  min_duration = 60,
  cmdline = unindent(
    r"""
    set -x
    until [[ -e _s3restore_done ]]; do
      sleep 60;
    done

    while sleep 60; do
      aws s3 sync {{src}} {{dst}} {{opts}}
    done
    """)).bind(opts='')

proc_prometheus = Process(
  name = 'prometheus',
  cmdline = unindent(
    r"""
    set -x
    set -e

    mkdir -p target_groups

    cat > target_groups/mesos.yml <<'EOF'
    - labels:
        service: mesos_exporter
      targets:
        - localhost:{{thermos.ports[mesos_exporter]}}
    EOF

    cat > target_groups/aurora.yml <<'EOF'
    - labels:
        service: aurora_exporter
      targets:
        - localhost:{{thermos.ports[aurora_exporter]}}
    EOF

    if ! type nix-env 2>/dev/null; then
      export PATH=/nix/var/nix/profiles/default/bin:$PATH
    fi
    export NIX_REMOTE=daemon
    nix-store -r --add-root ".gc/{{name}}" --indirect "{{Nix[prometheus]}}"
    export PATH="$PWD/.gc/{{name}}/bin:$PWD/.gc/{{name}}/sbin:$PATH"

    cp -RL "{{Nix[configs]}}"/{console_libraries,consoles,rules,prometheus.yml} .

    exec prometheus {{args}}
    """)
  ).bind(
    args = go_options({
      "web.listen-address": "0.0.0.0:{{thermos.ports[http]}}",
      "storage.local.path": "$PWD/metrics",
      "alertmanager.url": "http://localhost:{{thermos.ports[alertmanager]}}",
    }))

proc_s3restore = Process(
  name = 's3restore',
  cmdline = unindent(
    r"""
    set -e
    aws s3 sync {{src}} {{dst}} {{opts}}
    touch _s3restore_done
    """)).bind(opts='')

proc_mesos_exporter = simple_nixproc.bind(
  binary = 'mesos_exporter',
  store_path = '{{Nix[mesos_exporter]}}',
  args = go_options({
    'exporter.discovery.master-url': 'http://master.va.mesos.folsomlabs.com:5050',
    'exporter.discovery': '',
    'web.listen-address': ':{{thermos.ports[mesos_exporter]}}',
    'logtostderr': '',
    'v': 10
  }))

proc_aurora_exporter = simple_nixproc.bind(
  binary = 'aurora_exporter',
  store_path = '{{Nix[aurora_exporter]}}',
  args = go_options({
    'exporter.aurora-url': '{{aurora_url}}',
    'web.listen-address': ':{{thermos.ports[aurora_exporter]}}',
    'logtostderr': '',
  }))

proc_alertmanager = simple_nixproc.bind(
  binary = 'alertmanager',
  store_path = '{{Nix[alertmanager]}}',
  args = go_options({
    'config.file': '{{Nix[configs]}}/alertmanager.conf',
    'web.listen-address': ':{{thermos.ports[http]}}',
    'notification.smtp.sender': 'alerts@folsomlabs.com',
  }))

proc_mksynapse = simple_nixproc.bind(
  binary = 'mksynapse',
  store_path = '{{Nix[mksynapse]}}',
  args = '--stats {{thermos.ports[mksynapse]}} {{services}}')

PrometheusTask = Task(
  name = 'prometheus',
  resources = Resources(ram = 2*GB, cpu = 0.2, disk = 1*GB),
  processes = [
    proc_prometheus,
    proc_gentargets,
    proc_s3restore.bind(src='{{s3_backup_url}}/',
                        dst='.',
                        opts="--exclude='*' --include='metrics/*'"),
    proc_s3checkpoint.bind(src='./metrics',
                           dst='{{s3_backup_url}}/metrics'),
    proc_mesos_exporter,
    proc_aurora_exporter,
    proc_mksynapse.bind(
        services=('{{thermos.ports[alertmanager]}}'
                  ':{{a.role}}/{{a.environment}}/{{a.name}}'
                  ':http'))
  ],
  constraints = (order(proc_s3restore, proc_prometheus) +
                 order(proc_s3restore, proc_s3checkpoint))
  )

AlertmanagerTask = Task(
  name = 'alertmanager',
  resources = Resources(ram = 64*MB, cpu = 0.1, disk = 1*GB),
  processes = [
    proc_alertmanager,
    proc_s3restore.bind(src='{{s3_backup_url}}/alertmanager', dst='.'),
    proc_s3checkpoint.bind(src='.', dst='{{s3_backup_url}}/alertmanager',
                           opts="--exclude '*' --include silences.json")
  ],
  constraints = (order(proc_s3restore, proc_alertmanager) +
                 order(proc_s3restore, proc_s3checkpoint))
  )

ProdJobTemplate = Service(
  cluster = 'folsom',
  role = 'www-data',
  environment = 'prod',
  announce = Announcer(primary_port = 'http')
  ).bind(
    aurora_url = 'https://aurora.corp.folsomlabs.com',
    s3_backup_url = 's3://databucket/data/prometheus')

AlertmanagerJob = ProdJobTemplate(
  name = 'alertmanager',
  task = AlertmanagerTask)

PrometheusJob = ProdJobTemplate(
  name = 'prometheus',
  task = PrometheusTask)

jobs = [
  PrometheusJob.bind(a = AlertmanagerJob),
  AlertmanagerJob,
]

util.aurora.py

# Handy helpers for aurora files

import json
import textwrap
#import yaml

# De-indent a string and strip leading/trailing whitespace.
unindent = lambda x: textwrap.dedent(x).strip()

# Generate varname='varvalue' shell environment assignments from a dictionary
genShellEnv = lambda **x: " \\\n  ".join("%s='%s'" % (k, v) for k, v in x.items())

to_json = lambda x: json.dumps(x, indent=2)

# the aurora2 client doesn't seem to include libyaml.
#to_yaml = lambda x: yaml.dumps(x, indent=2)

# Like SimpleTask, but for things that don't need 1gb ram and a whole cpu.
def TinyTask(name, command):
    return Task(
        resources=Resources(ram = 512*MB, cpu = 0.1, disk = 100*MB),
        processes=[Process(name=name, cmdline=command)])