sample bulidbot confguration

master.cfg.py

# -*- python -*-
# vim: ts=4 sw=4 sts=4 expandtab syntax=python

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
import os
__basedir__ = os.path.abspath(os.path.dirname(__file__))

CCACHE_DIR = os.path.join(__basedir__, "ccache")
DIST_DIR = '/srv/build/out'
NIGHTLY_DIR = '/srv/files/nightly'

c = BuildmasterConfig = {}

####### BUILDSLAVES

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave
slaves = ["arm2_6dq", ]
passwd = 'passwd'
c['slaves'] = [BuildSlave(i, passwd, max_builds=1) for i in slaves]


# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# --master option)
c['slavePortnum'] = 9989

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot clone of pyflakes.

manifest_url = "gerrit:/platform/manifest.git"
# server defined  in $HOME/.ssh/config
gerrit_server = "gerrit-buildbot"
gerrit_user = "buildbot"

repo_branch = "r13.4_upstream"
moniter_branch = ("r13.3_upstream", "r13.4_upstream", "project", )

rules =  ("platform", "platform-lib-docs", "cts")

target_board = slaves

build_branches = []
for board in ("arm2_6dq", ):
    build_branches.append([board, rules, "default.xml", repo_branch])

DEPENDS_CHANGE_ID_TAG = "Depends-Change-Id: "
import subprocess
from buildbot.util import json
from buildbot.changes.gerritchangesource import GerritChangeSource
class ProjectChangeSource(GerritChangeSource):
    def eventReceived_change_merged(self, properties, event):
        change = event["change"]

        if "submitter" in event:
            author="%s <%s>" % (event["submitter"]["name"], event["submitter"]["email"])
        else:
            author="%s <%s>" % (change["owner"]["name"], change["owner"]["email"])

        return self.addChange(dict(
                author=author,
                project=change["project"],
                repository="ssh://%s@%s:%s/%s" % (
                    self.username, self.gerritserver, self.gerritport, change["project"]),
                branch=change["branch"],
                revision=event["patchSet"]["revision"],
                revlink=change["url"],
                comments=change["subject"],
                files=["unknown"],
                category=event["type"],
                properties=properties))

    def eventReceived_ref_updated(self, properties, event):
        return GerritChangeSource.eventReceived_ref_updated(self,
                properties, event)

    def eventReceived_patchset_created(self, properties, event):
        change = event["change"]
        dependChanges = self._getDependChanges(change["number"])
        if not dependChanges:
            return GerritChangeSource.eventReceived_patchset_created(self,
                    properties, event)

        from twisted.python import log
        dependDownloads = []
        for c in dependChanges:
            result = self._queryGerrit(c, includesCurrentPatchSet=True,
                    inJsonFormat=True)
            for line in result.split('\n'):
                try:
                    queryResult = json.loads(line.decode('utf-8'))
                except:
                    log.msg("Json fails to parse: " + queryResult.__str__())
                    continue
                if "project" in queryResult and "number" in queryResult:
                    # repo download <project> <number>[/patchset]
                    dependDownloads.append("%s %s/%s" %
                            (queryResult["project"],
                             queryResult["number"],
                             queryResult["currentPatchSet"]["number"]))
        if dependDownloads:
            log.msg("Adding another changes to check dependency: " + dependDownloads.__str__())
            downloads = properties.get("repo_downloads", [])
            downloads = list(set(downloads + dependDownloads))
            properties["repo_downloads"] = downloads

        # /usr/local/lib/python2.7/dist-packages/buildbot/changes/
        # gerritchangesource.py::eventReceived_patchset_created()
        return self.addChange(dict(
                    author="%s <%s>" % (change["owner"]["name"], change["owner"]["email"]),
                    project=change["project"],
                    repository="ssh://%s@%s:%s/%s" % (
                        self.username, self.gerritserver, self.gerritport, change["project"]),
                    branch=change["branch"]+"/"+change["number"],
                    revision=event["patchSet"]["revision"],
                    revlink=change["url"],
                    comments=change["subject"],
                    files=["unknown"],
                    category=event["type"],
                    properties=properties))

    def _queryGerrit(self, query,
            includesCommitMessage=False,
            includesCurrentPatchSet=False,
            inJsonFormat=False):
        # From /usr/local/lib/python2.7/dist-packages/buildbot/changes/
        # gerritchangesource.py::startStreamProcess()
        args = ["ssh", self.username + "@" + self.gerritserver,
                "-p", str(self.gerritport), "gerrit", "query", query]
        if includesCommitMessage:
            args += ["--commit-message"]
        if includesCurrentPatchSet:
            args += ["--current-patch-set"]
        if inJsonFormat:
            args += ["--format=JSON"]
        try:
            return subprocess.check_output(args)
        except Exception:
            from twisted.python import log
            log.msg("Exception happens during subprocess.check_output(%s)" % args)
            return -1

    def _getDependChanges(self, change):
        # TODO: Needs to get recursively
        changes = []
        result = self._queryGerrit(change, includesCommitMessage=True)
        if result != -1:
            for line in result.split('\n'):
                line = line.strip()
                if line.startswith(DEPENDS_CHANGE_ID_TAG):
                    changeId = line.strip().split(DEPENDS_CHANGE_ID_TAG)
                    if len(changeId) == 2:
                        changes.append(changeId[1])
        return changes

c['change_source'] = ProjectChangeSource(gerrit_server, gerrit_user)

####### SCHEDULERS

from buildbot.scheduler import Scheduler
from buildbot.schedulers.forcesched import ForceScheduler, UserNameParameter, ChoiceStringParameter, StringParameter
from buildbot.schedulers.timed import Nightly
from buildbot.changes.filter import ChangeFilter

PROJ_UBOOT =  'platform/bootable/bootloader/uboot-imx'
PROJ_KERNEL = 'kernel_imx'
UBOOT_BUILDER = '%s_uboot'
KERNEL_BUILDER = '%s_kernel'

def uboot_filter_fn(change):
    if not change.category in ('patchset-created', ):
        return

    branch = change.branch
    if '/' in branch:
        branch = branch.split('/', 1)[0]

    if not branch in moniter_branch:
        return

    if change.project == PROJ_UBOOT:
        return True

def kernel_filter_fn(change):
    if not change.category in ('patchset-created', ):
        return

    branch = change.branch
    if '/' in branch:
        branch = branch.split('/', 1)[0]

    if not branch in moniter_branch:
        return

    if change.project == PROJ_KERNEL:
        return True


def verify_filter_fn(change):
    #if not change.category in ('change-merged', 'patchset-created'):
    if not change.category in ('patchset-created', ):
        return
    branch = change.branch
    if '/' in branch:
        branch = branch.split('/', 1)[0]

    if not branch in moniter_branch:
        return

    if change.project == 'platform/manifest':
        return
    if change.project in (PROJ_KERNEL, PROJ_UBOOT):
        return

    if change.project.startswith('platform'):
        return True
    if change.project.startswith('vendor'):
        return True
    if change.project.startswith('device'):
        return True

c['schedulers'] = []
c['schedulers'].append(Scheduler(name="all", branch=None,
                                 treeStableTimer=2*60,
                                 builderNames=target_board))

c['schedulers'].append(Scheduler(name='project_verify',
                    treeStableTimer=60,
                    change_filter = ChangeFilter(filter_fn = verify_filter_fn),
                    builderNames= [x + '_verify' for x in target_board]))

c['schedulers'].append(Scheduler(name='project_uboot',
                    treeStableTimer=2*60,
                    change_filter = ChangeFilter(filter_fn = uboot_filter_fn),
                    builderNames= [UBOOT_BUILDER%x for x in target_board]))

c['schedulers'].append(Scheduler(name='project_kernel',
                    treeStableTimer=2*60,
                    change_filter = ChangeFilter(filter_fn = kernel_filter_fn),
                    builderNames= [KERNEL_BUILDER%x for x in target_board]))

c['schedulers'].append(Nightly(name='project_nightly',
                    dayOfWeek=range(5),
                    branch = repo_branch,
                    hour=(23, ),
                    minute=(0, ),
                    builderNames = target_board))

c['schedulers'].append(ForceScheduler(name='force',
        branch = ChoiceStringParameter(name="branch",
            choices=[repo_branch, ], default=repo_branch),
        username=UserNameParameter(label="your name:<br>", size=80),
        properties = [
            StringParameter(name="repo_d", size=80),
            StringParameter(name="repo_d0", size=80),
            StringParameter(name="repo_d1", size=80),
            StringParameter(name="repo_d2", size=80),
        ],
        builderNames = target_board))


####### BUILDERS


from buildbot.process import factory
from buildbot.steps.source import Repo
from buildbot.steps.shell import Compile
from buildbot.steps.master import  MasterShellCommand
from buildbot.steps.transfer import FileUpload
from buildbot.steps.python_twisted import Trial
from buildbot.config import BuilderConfig
from buildbot.process.properties import WithProperties

def getBaseCommand(board):
    return """
            set -e
            export LC_ALL=C
            export LC_MESSAGES=C
            export LANGUAGE=C
            export LANG=C
            if [ -f vendor/product/build/setup.sh ]; then
                . vendor/product/build/setup.sh
                export CCACHE_DIR=%s
                envsetup %s userdebug
            fi
"""%(CCACHE_DIR, board)


def build_factory(board, rules, clean = False, f1 = None):
    manifest_file = 'default.xml'
    if not f1:
        f1 = factory.BuildFactory()
        f1.workdir="build"

    basecommand = getBaseCommand(board)
    preparecommand = basecommand + """
    mount | grep $PWD/$OUT_DIR && umount $PWD/$OUT_DIR || true
    mount
    """

    f1.addStep(Repo(manifest_url = manifest_url,
            manifest_branch = repo_branch,
            manifest_file = manifest_file,
            alwaysUseLatest = True,
            mode = "update" ))

    f1.addStep(Compile(name="prepare directory",
            description = ["preparing directory"],
            descriptionDone = ["prepare directory"],
            command=["/bin/bash","-c", preparecommand]))

    cleancommand = basecommand + """
        mount | grep $PWD/$OUT_DIR || mkdir -p $PWD/$OUT_DIR && mount $PWD/$OUT_DIR || true
        mount
    """

    if clean:
        cleancommand += """
            echo rm -rf "$OUT_DIR"/*
            rm -rf "$OUT_DIR"/*
            echo rm -rf %(dist)s/*
            rm -rf %(dist)s/*
        """%{'dist': DIST_DIR}
    else:
        cleancommand += """
            echo rm -rf "$OUT"/{system,root,data}
            rm -rf "$OUT"/{system,root,data}
            echo rm -rf "$OUT"/obj/{BOOTLOADER_OBJ,KERNEL_OBJ}
            rm -rf "$OUT"/obj/{BOOTLOADER_OBJ,KERNEL_OBJ}
            echo rm -rf "$OUT_DIR"/target/common/obj/JAVA_LIBRARIES/{framework,product}_intermediates/src/*
            rm -rf "$OUT_DIR"/target/common/obj/JAVA_LIBRARIES/{framework,product}_intermediates/src/*
        """
    f1.addStep(Compile(name="clean up",
            description = ["cleaning\ngenerated codes"],
            descriptionDone = ["cleaned up\ngenerated codes"],
            command=["/bin/bash","-c", cleancommand]))

    buildcommand = basecommand + """
            mkkernel imx6_mm2014_defconfig include/linux/version.h
            make -j7 %s \
                HOST_CC="prebuilt/linux-x86/ccache/ccache gcc-4.5" \
                HOST_CXX="prebuilt/linux-x86/ccache/ccache g++-4.5" \
            """%(" ".join(rules))

    f1.addStep(Compile(name="compile android",
                description = ["compiling android"],
                descriptionDone = ["compile android"],
                command=["/bin/bash","-c", buildcommand]))

    return f1


builders = []
for board in target_board:
    nightly_rules = rules + ("dist", '='.join(("DIST_DIR", DIST_DIR)))
    f1 = build_factory(board, nightly_rules, True)

    # todo should upload result of compilation somewhere else
    basecommand = getBaseCommand(board)

    ubootcommand = basecommand + """
    mkuboot
    cp -f $OUT/u-boot.bin %(dist)s/
    """%{'dist': DIST_DIR}
    f1.addStep(Compile(name="compile uboot",
                description = ["compiling uboot"],
                descriptionDone = ["compile uboot"],
                command=["/bin/bash","-c", ubootcommand]))
 
    copycommand = basecommand + """
            cp -af "$OUT_DIR"/target/common/docs/ivi-lib/* %s
"""%(os.path.join(__basedir__, "public_html"))
    f1.addStep(Compile(name="copy documents",
                description = ["copying documents"],
                descriptionDone = ["copy documents"],
                command=["/bin/bash","-c", copycommand]))
    
    uploadcommand = """
            cp -f manifest-original.xml %(dist)s/manifest.xml
            DATETIME=$(date +"%%Y-%%m-%%d %%H.%%M")
            REMOTE="%(remote)s/${DATETIME}"
            lftp -e "lcd %(dist)s && mkdir -p '${REMOTE}' && cd '${REMOTE}' && mput *" sftp://storage 
"""%{'dist': DIST_DIR, 'remote': NIGHTLY_DIR}
    f1.addStep(Compile(name="upload image",
                description = ["uploading image"],
                descriptionDone = ["upload image"],
                command=["/bin/bash","-c", uploadcommand]))

    buildname ="%s"%(board)
    b1 = BuilderConfig(name = buildname,
               slavenames = slaves,
               builddir = board,
               factory = f1)
    builders.append(b1)

    f1 = build_factory(board, rules)
    buildname ="%s_verify"%(board)
    b1 = BuilderConfig(name = buildname,
               slavenames = slaves,
               builddir = buildname,
               factory = f1)
    builders.append(b1)

    f1 = build_factory(board, ['uboot',])

    buildname = UBOOT_BUILDER%board
    b1 = BuilderConfig(name = buildname,
               slavenames = slaves,
               builddir = buildname,
               factory = f1)
    builders.append(b1)

    f1 = build_factory(board, ['uImage',])

    buildname = KERNEL_BUILDER%board
    b1 = BuilderConfig(name = buildname,
               slavenames = slaves,
               builddir = buildname,
               factory = f1)
    builders.append(b1)



c['builders'] = builders

####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

# workaround
import twisted.web.error
import twisted.web.resource
twisted.web.error.NoResource = twisted.web.resource.NoResource

from buildbot.status import html
from buildbot.status.web import auth, authz
from buildbot.status.status_gerrit import GerritStatusPush
from buildbot.status.builder import Results, SUCCESS, RETRY, FAILURE, WARNINGS

authz_cfg=authz.Authz(
    # change any of these to True to enable; see the manual for more
    # options
    gracefulShutdown = True,
    forceBuild = True,
    forceAllBuilds = True,
    pingBuilder = True,
    stopBuild = True,
    stopAllBuilds = True,
    cancelPendingBuild = True,
)
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))

def gerritMessageCB(buildername, build, results, status, arg):
    if results == RETRY:
        return None, 0, 0
    sep="\n\n"
    message = "buildbot finished compiling your patchset\n"
    message += sep
    message += "on configuration %s\n"%(buildername)
    message += sep
    message += "the result is %s\n"%(Results[results])
    message += sep
    message += "more details %sbuilders/%s/builds/%d\n"%(c['buildbotURL'],buildername,build.getNumber())
    return message, (results == SUCCESS or -1), 0

class GerritResultPush(GerritStatusPush):
    def __init__(self, *args, **kwds):
        from twisted.python import log
        GerritStatusPush.__init__(self, *args, **kwds)
        log.msg("Gerrit Result Push initialized. %s, %s, %s" %
                (self, args, kwds))

    def startService(self):
        from twisted.python import log
        GerritStatusPush.startService(self)
        log.msg("Gerrit Result Push Service Started. %s" % self)

    def buildFinished(self, builderName, build, result):
        from twisted.python import log
        log.msg("build finished. preparing send result to gerrit %s %s %s %s" %
                (self, builderName, build, result))
        """Do the SSH gerrit verify command to the server."""

        # Gerrit + Repo
        downloads = build.getProperty("repo_downloads")
        downloaded = build.getProperty("repo_downloaded")
        log.msg("downloads = ", downloads, "downloaded = ", downloaded)
        if downloads is not None and downloaded is not None:
            downloaded = downloaded.split(" ")
            if downloads and 2 * len(downloads) == len(downloaded):
                log.msg("run review callback ")
                message, verified, reviewed = self.reviewCB(builderName, build, result, self.status, self.reviewArg)
                log.msg("message = ", message)
                if message is None:
                    return
                for i in range(0, len(downloads)):
                    try:
                        project, change1 = downloads[i].split(" ")
                    except ValueError:
                        log.msg("value error")
                        return # something is wrong, abort
                    change2 = downloaded[2 * i]
                    revision = downloaded[2 * i + 1]
                    log.msg("change2 = " + change2 + ", revision = " + revision)
                    if change1 == change2:
                        self.sendCodeReview(project, revision, message, verified, reviewed)
                    else:
                        log.msg("changes is invalid")
                        return # something is wrong, abort
            else:
                log.msg("download count is invalid")
            return

    from twisted.internet.protocol import ProcessProtocol
    class LocalPP(ProcessProtocol):
        def __init__(self, status):
            self.status = status

        def outReceived(self, data):
            from twisted.python import log
            log.msg("gerritout:", data)

        def errReceived(self, data):
            from twisted.python import log
            log.msg("gerriterr:", data)

        def processEnded(self, status_object):
            from twisted.python import log
            if status_object.value.exitCode:
                log.msg("gerrit status: ERROR:", status_object)
            else:
                log.msg("gerrit status: OK")

    def sendCodeReview(self, project, revision, message=None, verified=0, reviewed=0):
        from twisted.python import log
        log.msg("send result to gerrit %s %s %s %s %s %s" %
                (self, project, revision, message, verified, reviewed))
        GerritStatusPush.sendCodeReview(self, project, revision, message, verified, reviewed)


c['status'].append(GerritResultPush(gerrit_server,gerrit_user,gerritMessageCB))


buildbot_addr ="[email protected]"
build_log_addr = '[email protected]'
mail_receivers = [buildbot_addr, build_log_addr]
#mail_receivers = [buildbot_addr, ]
buildbot_relayhost = "smtp.googlemail.com"
buildbot_passwd = "blahblah"

from buildbot.status import mail

def build_fail_message_formatter(mode, name, build, results, master_status):
    body = mail.defaultMessage(mode, name, build, results, master_status)['body']

    return {'body': body, 'type': 'plain'}

c['status'].append(mail.MailNotifier(fromaddr=buildbot_addr,
                extraRecipients=mail_receivers,
                useTls=True,
                relayhost=buildbot_relayhost,
                smtpPort=587,
                smtpUser=buildbot_addr,
                smtpPassword=buildbot_passwd,
                mode=['failing', 'change'],
                sendToInterestedUsers=True))

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "Some Project"
c['titleURL'] = "http://blahblah.com"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "http://buildbot.blahblah.com:8010/"

####### DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can leave
    # this at its default for all but the largest installations.
    'db_url' : "sqlite:///state.sqlite",
}

getOutputDir = WithProperties(os.path.join(__basedir__, "..", "builds", "build-%s-%s"),"buildername","changenumber")
del os
getWebDir = WithProperties(c['buildbotURL'] + "builds/build-%s-%s","buildername","changenumber")