jenkins.py

#!/usr/bin/env python
import argparse
import subprocess
import os
import os.path
import multiprocessing
import sys
import shutil
import glob

# Expect:
# workdir
#   cbsdkd
#   sdkd-cpp
#   libcouchbase

CLUSTER_CONFIG_TEMPLATE = \
'''
[global]
username:root
password:couchbase
port:8091

[servers]
{SERVERS}

[membase]
rest_username:Administrator
rest_password:password
'''

BUILD_ID = os.environ.get('BUILD_ID', 'NOT_JENKINS')

def write_cluster_config(nodes, target):
    fp = open(target, "w")
    server_lines = []
    for ix, node in zip(range(1, 100), nodes):
        server_lines.append("{0}:{1}".format(ix, node))

    fp.write(CLUSTER_CONFIG_TEMPLATE.format(SERVERS="\n".join(server_lines)))
    fp.flush()
    fp.close()

def make_target(tstr=""):
    ret = "make -s -j {ncpu} {tgt}".format(
        ncpu = multiprocessing.cpu_count() + 1,
        tgt = tstr)
    return ret

def invoke_shell(cmd):
    sys.stderr.write("Invoking.. " + cmd + "\n")
    sys.stderr.flush()
    po = subprocess.Popen(("bash"), stdin=subprocess.PIPE)
    po.communicate("set -e;" + cmd)
    assert po.returncode == 0

def get_lcb_version():
    curdir = os.getcwd()
    os.chdir(os.path.join(WORKDIR, 'libcouchbase'))
    retval = None

    try:
        po = subprocess.Popen(
            ('git', 'describe', '--long', '--tags'),
            stdout=subprocess.PIPE)

        out, err = po.communicate()
        assert po.returncode == 0

        retval = out.rstrip().strip()

    finally:
        os.chdir(curdir)

    return retval


def create_sdkd_wrappers(sdkd_path,
                         sdkd_target,
                         argfile_target,
                         core_target):
    script_txt = r'''#!{py}
import os
import sys
import os.path
import subprocess
import glob
import shutil

sys.stderr.write(
    "Using build number " +
    os.environ.get("BUILD_ID", "NOT_JENKINS") +
    '\n')

sys.stderr.flush()
curdir = os.getcwd()

po = subprocess.Popen([
    r'{sdkd}',
    "-l", "4444",
    "--ttl", "2000"
    ])
po.wait()

# See if there's a 'core' file.
for core in glob.glob(os.path.join(curdir, "*core*")):
    sys.stderr.write("Found core " + core + r"\n")
    corebase = os.path.basename(core)
    corebase += "." + str(po.pid)
    corebase = os.path.join(r"{core_target}", corebase)
    shutil.move(core, corebase)

    '''.format(sdkd=sdkd_path, py=sys.executable, core_target=core_target)

    argfile_txt = '''\
    -C share/rexec \
    --rexec_path {sdkd_target} \
    --rexec_port 4444
    '''.format(sdkd_target=sdkd_target)


    fp = open(sdkd_target, "w")
    fp.write(script_txt)
    fp.close()
    os.chmod(sdkd_target, 0777)

    fp = open(argfile_target, "w")
    fp.write(argfile_txt)
    fp.close()


def check_upload_coredumps():
    if not os.path.exists(PERSISTENT_COREDIR):
        os.mkdir(PERSISTENT_COREDIR)

    run_dst = os.environ.get("BUILD_TAG", "NO-JENKINS") + "-COREDUMPS"
    run_dst = os.path.join(PERSISTENT_COREDIR, run_dst)
    if glob.glob(os.path.join(COREDUMPS, "*core*")):
        shutil.copytree(INSTDIR, run_dst)
        shutil.copytree(COREDUMPS, os.path.join(run_dst, "CORES"))
        print "Core at " + run_dst


ALLOWED_STEPS = [
    "lcb", # build libcouchbase
    "sdkd", # build sdkd-cpp
    "cbsdkd", # setup test environment
    "cluster-install", # install the cluster
    "test", # run the test
    "report-upload", # upload the report to google docs
]

ap = argparse.ArgumentParser()

ap.add_argument('-W', '--workdir',
                help="Working root directory", required=True)

ap.add_argument('-S',
                '--steps', help="Command Step",
                choices=ALLOWED_STEPS,
                action='append', default=[])

ap.add_argument('-N', '--nodes',
                help="Cluster nodes to use (hostname only)",
                action='append')

ap.add_argument('-V', '--version',
                help="Cluster version to install",
                default='2.1.0')

ap.add_argument('--single-test',
                help="Specify single test to run (quick)")

ap.add_argument('--cluster-config',
                help="Alternate cluster configuration file")

opts, BRUN_USER_ARGS = ap.parse_known_args()

WORKDIR = opts.workdir
SDKD_SCRIPT = os.path.join(WORKDIR, "SDKD_WRAP")
SDKD_ARGFILE = os.path.join(WORKDIR, 'LCB.args')

CLUSTER_INI = opts.cluster_config or os.path.join(WORKDIR, "cluster.ini")
CLUSTER_INI = os.path.abspath(CLUSTER_INI)

TESTLISTS = os.path.join(WORKDIR, "testdefs")
TESTLIST_HYBRID = os.path.join(TESTLISTS, "tests-all")
INSTDIR = os.path.join(WORKDIR, 'INST')
LCBDIR = os.path.join(WORKDIR, 'libcouchbase')
SDKDDIR = os.path.join(WORKDIR, 'sdkd-cpp')
CBSDKDDIR = os.path.join(WORKDIR, 'cbsdkd')
SDKD_PATH = os.path.join(INSTDIR, 'bin', 'sdkd_lcb')
COREDUMPS = os.path.join(WORKDIR, "COREDUMPS")
PERSISTENT_COREDIR = os.path.join(WORKDIR, "..")
XUNIT_DIR = os.path.join(WORKDIR, 'XUNIT_LOGS')
LOG_DIR = os.path.join(WORKDIR, 'LOGS_' + BUILD_ID)
SPREADSHEET_NAME = None
LCB_VERSION = None

os.environ['CBSDKD_FORCE_COLOR'] = "1"
os.chdir(opts.workdir)



if not opts.nodes:
    opts.nodes = ['10.3.4.7', '10.3.3.249', '10.3.4.14']


if 'lcb' in opts.steps:
    if os.path.exists(INSTDIR):
        shutil.rmtree(INSTDIR)

    invoke_shell(
    '''
    cd libcouchbase

    git clean -qdfx && git reset --hard

    ./config/autorun.sh
    ./configure --prefix={inst} --enable-debug --enable-warnings --silent --disable-tools --disable-examples

    {make} V=0 > /dev/null
    {make} check V=0
    {make} install V=0 > /dev/null
    '''.format(inst=INSTDIR, make=make_target())
    )

if 'sdkd' in opts.steps:
    invoke_shell(
    '''
    cd sdkd-cpp
    git clean -dfx && git reset --hard

    git submodule init
    git submodule update
    ''')

    invoke_shell(
    '''
    cd sdkd-cpp/src/contrib/json-cpp
    python amalgamate.py
    ''')

    invoke_shell(
    '''
    cd sdkd-cpp
    rm -rf build || true
    mkdir build
    cd build
    cmake -DLCB_ROOT={inst} -DCMAKE_BUILD_TYPE=DEBUG -DCMAKE_INSTALL_PREFIX={inst} ../
    {make} install
    '''.format(inst=INSTDIR, make=make_target()))


if 'cbsdkd' in opts.steps:
    invoke_shell('''
    cd cbsdkd
    git submodule init
    git submodule update
    ''')

    invoke_shell('''
    cd cbsdkd
    rm -f test.db
    rm -rf batch || true
    ./dbmanage create_db
    python share/db/std-tests.py -d {testlists} -o {wdir}/testdefs.json
    ./dbmanage reg_scenarios -f {wdir}/testdefs.json
    '''.format(wdir=WORKDIR, testlists=TESTLISTS))


    # Now, create the .ini file
    if not opts.cluster_config:
        write_cluster_config(opts.nodes, CLUSTER_INI)

if 'cluster-install' in opts.steps:
    invoke_shell('''
    cd cbsdkd/contrib/testrunner
    ./scripts/install.py -i {ini} -p version={clvers},product=cb,parallel=True,vbuckets=128
    '''.format(ini=os.path.join(WORKDIR, "cluster.ini"),
               clvers=opts.version))



if 'test' in opts.steps:
    # So, we've done all the package setup stuff. Now it's time to actually invoke
    # the SDKD..

    # .. get the 'git describe' output

    if not os.path.exists(LOG_DIR):
        os.makedirs(LOG_DIR)
    try:
        os.unlink(XUNIT_DIR)
    except OSError:
        pass

    os.symlink(LOG_DIR, XUNIT_DIR)

    try:
        LCB_VERSION = get_lcb_version()
        os.chdir(os.path.join(WORKDIR, 'cbsdkd'))
        sdk_existing = subprocess.Popen(('python', 'dbmanage', 'list_sdks'),
                                        stdout=subprocess.PIPE
                                        ).communicate()[0]

        if sdk_existing.find(LCB_VERSION) == -1:
            invoke_shell(
                "./dbmanage reg_sdk -t C -n {vers} -r 2.x".format(vers=LCB_VERSION)
            )

    finally:
        os.chdir(WORKDIR)

    if not os.path.exists(COREDUMPS):
        os.mkdir(COREDUMPS)

    # Generate the BRUN commandline. First though, we need to generate an
    # argfile to invoke the SDKD
    create_sdkd_wrappers(SDKD_PATH,
                         SDKD_SCRIPT,
                         SDKD_ARGFILE,
                         COREDUMPS)


    BRUN_OPTS = [
        '--sdkd-config', SDKD_ARGFILE,
        '--sdk', LCB_VERSION,
        '--ini', CLUSTER_INI,
        '--max-age', '1s',
        '--log-dir', LOG_DIR
        #'--no-upload',
    ] + BRUN_USER_ARGS

    if opts.single_test:
        BRUN_OPTS += [ '--test', opts.single_test ]
    else:
        BRUN_OPTS += [ '--test-list', TESTLIST_HYBRID]

    os.chdir(CBSDKDDIR)
    po = subprocess.Popen(['python', 'brun'] + BRUN_OPTS)
    po.wait()
    check_upload_coredumps()
    assert po.returncode == 0


if 'report-upload' in opts.steps:
    os.chdir(os.path.join(WORKDIR, 'cbsdkd'))
    invoke_shell('./report -S {0}'.format(get_lcb_version()))
    sdir = os.path.join(WORKDIR, "cbsdkd", "batch", "xl")
    spreadsheet = glob.glob(os.path.join(sdir, "*.xlsx"))[0]
    print "Have " + spreadsheet