Auto build and configure a multi-master mesos cluster

auto-mesos-config.py

#!/usr/bin/python3.4

from datetime import datetime
from shutil import copy2
from os import makedirs
from os.path import exists
import socket
import fileinput
import re
import math
import subprocess

def service_running(service_name):
    command = "status {0} | grep -q \"^{1} start\" > /dev/null".format(service_name, service_name)
    if subprocess.call(command, shell=True) == 0:
        return True
    else:
        return False

def start_or_restart_service(service):
    if service_running(service):
        if subprocess.call("restart {0}".format(service), shell=True):
            print("successfully restarted {0}".format(service))
        else:
            print("error: failed to restart {0}".format(service))
    else:
        if subprocess.call("start {0}".format(service), shell=True):
            print("successfully started {0}".format(service))
        else:
            print("error: failed to start {0}".format(service))

def backup_file(file):
    if exists(file):
        file_backup_name = "{0}{1}".format(file, file_backup_suffix)
        copy2(file, file_backup_name)
        print("backed up {0} to {1}".format(file, file_backup_name))
    else:
        print("{0} cannot be backed up because it doesn't exist".format(file))

def remove_pattern_from_file(filename, remove_pattern):
    matched = re.compile(remove_pattern).search
    with fileinput.FileInput(filename, inplace=1) as file:
        for line in file:
            if not matched(line):
                print(line, end='')

zk_id_file = "/etc/zookeeper/conf/myid"
zk_config_file = "/etc/zookeeper/conf/zoo.cfg"
mesos_quorum_file = "/etc/mesos-master/quorum"
mesos_master_zk_config_file = "/etc/mesos/zk"
mesos_ip_file = "/etc/mesos-master/ip"
mesos_hostname_file = "/etc/mesos-master/hostname"
marathon_conf_dir = "/etc/marathon/conf"
marathon_hostname_file = "/etc/marathon/conf/hostname"
marathon_mesos_zk_config_file = "/etc/marathon/conf/master"
marathon_zk_config_file = "/etc/marathon/conf/zk"
mesos_slave_override_file = "/etc/init/mesos-slave.override"

master_zk_port = 2181
zk_leader_port = 2888
zk_election_port = 3888
num_masters = 3

mesos_zk_root_path = "mesos"
marathon_zk_root_path = "marathon"

fqdn_pattern = re.compile('^([^\d]+)(\d+)-(master|slave)(\d+)\.(\S+)$');
fqdn_match = fqdn_pattern.match(socket.getfqdn())

now = datetime.utcnow()
file_backup_suffix = ".backup_{0}-{1}-{2}-{3}-{4}-{5}-{6}".format(
    now.year,
    now.month,
    now.day,
    now.hour,
    now.minute,
    now.second,
    now.microsecond)

print("file backup suffix is: {0}".format(file_backup_suffix))

if len(fqdn_match.groups()) != 5:
    print("Error: unable to parse the five name parts from fqdn {0}".format(socket.getfqdn()))
    exit(1)

cluster_name, cluster_num, role, role_num, domain = fqdn_match.groups();
print("fqdn: {0}".format(socket.getfqdn()))
print("cluster name: {0}".format(cluster_name))
print("cluster number: {0}".format(cluster_num))
print("role: {0}".format(role))
print("role number: {0}".format(role_num))
print("domain: {0}".format(domain))

master_zk_conn_str = "zk://"

for master_num in range(1, num_masters + 1):
    master_zk_conn_str = "{0}{1}{2}-master{3}.{4}:{5}".format(
        master_zk_conn_str,
        cluster_name,
        cluster_num,
        master_num,
        domain,
        master_zk_port)
    if master_num != num_masters:
        master_zk_conn_str = "{0},".format(master_zk_conn_str)

mesos_master_zk_conn_str = "{0}/{1}".format(master_zk_conn_str, mesos_zk_root_path)
backup_file(mesos_master_zk_config_file)
open(mesos_master_zk_config_file, 'w').write("{0}\n".format(mesos_master_zk_conn_str))
print("configured {0} with {1}".format(mesos_master_zk_config_file, mesos_master_zk_conn_str))

if role == "master":
    print("configuring mesos node as master {0}...".format(role_num))

    backup_file(zk_id_file)
    open(zk_id_file, 'w').write("{0}\n".format(role_num))
    print("updated id {0} in {1}".format(role_num, zk_id_file))

    backup_file(zk_config_file)
    remove_pattern = re.compile("server\.\d+=")
    remove_pattern_from_file(zk_config_file, remove_pattern)
    print("removed references to previous server config in {0}".format(zk_config_file))

    open_zk_config_file = open(zk_config_file, 'a')
    for master_num in range(1, num_masters + 1):
        master_zk_config = "{0}{1}-master{2}.{3}:{4}:{5}".format(
            cluster_name,
            cluster_num,
            master_num,
            domain,
            zk_leader_port,
            zk_election_port)
        config_line = "server.{0}={1}".format(master_num, master_zk_config)
        open_zk_config_file.write("{0}\n".format(config_line))
        print("wrote {0} to {1}".format(config_line, zk_config_file))

    backup_file(mesos_quorum_file)
    quorum_num = math.ceil(num_masters / 2)
    open(mesos_quorum_file, 'w').write("{0}".format(quorum_num))
    print("configured quorum count requirement for {0}", quorum_num)

    backup_file(mesos_ip_file)
    open(mesos_ip_file, 'w').write(socket.gethostbyname(socket.gethostname()))
    print("wrote {0} to {1}".format(socket.gethostbyname(socket.gethostname()), mesos_ip_file))

    backup_file(mesos_hostname_file)
    open(mesos_hostname_file, 'w').write(socket.getfqdn())
    print("wrote {0} to {1}".format(socket.getfqdn(), mesos_hostname_file))

    print("configuring marathon on master {0}...".format(role_num))
    makedirs(marathon_conf_dir, 0o755)
    print("ensured {0} exists".format(marathon_conf_dir))

    copy2(mesos_hostname_file, marathon_hostname_file)
    print("copied {0} to {1}".format(mesos_hostname_file, marathon_hostname_file))

    copy2(mesos_master_zk_config_file, marathon_mesos_zk_config_file)
    print("copied {0} to {1}".format(mesos_master_zk_config_file, marathon_mesos_zk_config_file))

    marathon_master_zk_conn_str = "{0}/{1}".format(master_zk_conn_str, marathon_zk_root_path)
    backup_file(marathon_zk_config_file)
    open(marathon_zk_config_file, 'w').write("{0}\n".format(marathon_master_zk_conn_str))
    print("wrote {0} to {1}".format(marathon_master_zk_conn_str, marathon_zk_config_file))

    if service_running("mesos-slave"):
        if subprocess.call("stop mesos-slave", shell=True) == 0:
            print("stopped service: mesos-slave")
        else:
            print("failed to stopped service: mesos-slave")
    open(mesos_slave_override_file, 'w').write("manual")
    print("mesos-slave mode configured with manual init (should not start on master)")

    start_or_restart_service("zookeeper")
    start_or_restart_service("mesos-master")
    start_or_restart_service("marathon")