build package with cowbuilder or git-buildpackage on Jenkins

debbuild.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
''' debbuild_wrapper.py
The debian package build wrapper on Jenkins.
This script support three way of building debian package.

1-1. Backport from source package with cowbuilder.
1-2. Pre-build and Backport from source package with cowbuilder.
2. Build original package with git-buildpackage.

Support flavors are "debian" or "ubuntu".
'''
import deb822
import shutil
import argparse
import subprocess
import pexpect
import os.path
import sys
import pwd
import shlex
from glob import glob
from pydebsign import debsign


def parse_options():
    """ building options """
    parser = argparse.ArgumentParser(description='usage')

    parser.add_argument('-c', '--codename', action='store', required=True,
                        help='specify codename of distribution')
    # for piuparts
    parser.add_argument('-f', '--flavor', choices=['debian', 'ubuntu'],
                        default='debian',
                        help='specify flavor debian or ubuntu (for piuparts)')
    parser.add_argument('-m', '--mirror', action='store',
                        help='specify Debian mirror (for piuparts)')
    # Backport version is different with an existing version in the arvhive,
    # so generally upgrade test fails.
    parser.add_argument('-n', '--no-upgrade-test', action='store_true',
                        help=('Skip testing upgrade from '
                              'an existing version in the archive.'))
    # without lintian checking
    parser.add_argument('--without-lintian', action='store_true',
                        help=('Skip checking with lintian'))

    # for dput
    parser.add_argument('-p', '--passphrase', action='store', required=True,
                        help='GPG private key passphrase of uploder')
    parser.add_argument('-r', '--reprepro-passphrase', action='store',
                        required=True,
                        help='GPG Private key passphrase of reprepro')

    subparser = parser.add_subparsers(title='subcommands',
                                      dest='subparser_name')
    backport = subparser.add_parser('backport', help='backport package')
    subparser.add_parser('original', help='original package')
    backport.add_argument('-u', '--url', action='store', required=True,
                          help='specify .dsc url')
    backport.add_argument('-P', '--prebuild', action='store_true',
                          help='pre-build debian package')
    backport.add_argument('--debfullname', action='store')
    backport.add_argument('--debemail', action='store')
    return parser.parse_args()


def set_work_dirpath():
    """ set work_dirpath variable
    expected value is WORKSPACE on Jenkins,
    current directory is on local test.
    """
    if os.environ.get('WORKSPACE'):
        work_dirpath = os.environ.get('WORKSPACE')
    else:
        work_dirpath = os.path.abspath(os.path.curdir)
    return work_dirpath


def make_directories(dir_list):
    """ make directories for results """
    [os.mkdir(dir_name) for dir_name in dir_list
     if os.path.isdir(dir_name) is False]


def parse_changes(changes_filepath):
    """ parse .changes file """
    with open(changes_filepath, 'rb') as changes_file:
        changes = deb822.Changes(changes_file)
    return [_file for _file in changes['Files']]


def get_architecture():
    """ get DEB_BUILD_ARCH """
    command = 'dpkg-architecture -qDEB_BUILD_ARCH'
    return subprocess.check_output(
        shlex.split(command)).split()[0].decode('utf-8')


def retrieve_source_package(dsc_url):
    """ retrieve source package with dget command """
    command = 'dget -d %s' % dsc_url
    return subprocess.call(shlex.split(command))


def get_changes_name(dsc_name, arch):
    """ get .changes file name """
    pkg_name = dsc_name.split('.dsc')[0]
    changes_name = '%s_%s.changes' % (pkg_name, arch)
    return changes_name


def get_basepath(codename):
    """ get cowbuilder basepath """
    return '/var/cache/pbuilder/base-%s.cow' % codename


def get_basetgz(codename):
    """ get pbuilder basetgz path """
    return '/var/cache/pbuilder/base-%s.tgz' % codename


def update_cowbuilder(basepath):
    """ update cowbuilder image """
    command = 'sudo /usr/sbin/cowbuilder --update --basepath %s' % basepath
    return subprocess.call(shlex.split(command))


def extract_source_package(dsc_path):
    """ extract source package and meta data from .control file """
    command = 'dpkg-source -x %s' % dsc_path
    subprocess.call(shlex.split(command))
    with open(dsc_path, 'rb') as dsc_file:
        srcs = deb822.Sources(dsc_file)

    build_depends = srcs.get('Build-Depends').split(',')
    if srcs.get('Build-Depends-Indep'):
        build_depends += (srcs.get('Build-Depends-Indep').split(','))

    data = {'source_dirname': '%s-%s' % (srcs.get('Source'),
                                         srcs.get('Version').split('-')[0]),
            'build_depends': build_depends,
            'dsc_name': os.path.basename(dsc_path),
            'upstream_version': srcs.get('Version').split('-')[0],
            'source_package_name': srcs.get('Source'),
            'debian_version': srcs.get('Version'),
            'batch_name': '%s-batch.sh' % srcs.get('Source')}
    return data


def append_param(param_dict, args):
    """ append variables to param_dict from args """
    param_dict['dsc_url'] = args.url
    param_dict['debfullname'] = args.debfullname
    param_dict['debemail'] = args.debemail
    return param_dict


def generate_batch_script(param_dict):
    """ generate batch script for cowbuilder """
    return ('#!/bin/sh -x\n'
            'export DEBFULLNAME="%(debfullname)s"\n'
            'export DEBEMAIL=%(debemail)s\n'
            'apt-get -y install curl devscripts quilt '
            'patch libdistro-info-perl fakeroot\n'
            'apt-get -y build-dep %(source_package_name)s\n'
            'dget -d %(dsc_url)s\n'
            'dpkg-source -x %(dsc_name)s\n'
            '(\n'
            'cd %(source_dirname)s\n'
            'debuild -us -uc\n'
            ')\n'
            'cp -f %(source_package_name)s_%(debian_version)s.debian.tar.gz '
            '%(source_package_name)s_%(upstream_version)s.orig.tar.gz '
            '%(source_package_name)s_%(debian_version)s.dsc '
            '%(temp_dirpath)s/\n'
            % param_dict)


def prepare_pbuilderrc(workspace_dirpath, codename):
    """ prepare .pbuilderrc file of cowbuilder for pre-building
    ---
    DISTRIBUTION: codename
    BINDMOUNTS: for "temp" results of pre-build
    BUILDRESULT: for "unsigned_results" of backports build
    """
    content = ('DISTRIBUTION="%s"\n'
               'BINDMOUNTS="%s/temp"\n'
               'DEBBUILDOPTS="-sa"\n'
               'BUILDRESULT="%s/unsigned_results"\n'
               % (codename, workspace_dirpath, workspace_dirpath))
    pbuilderrc_path = os.path.join(workspace_dirpath, '.pbuilderrc')
    with open(pbuilderrc_path, 'w') as _file:
        _file.write(content)
    return pbuilderrc_path


def pre_build_package(pbuilderrc_path, codename, basepath, batch_filepath):
    """ pre-building source package for backport """
    command = ('sudo /usr/sbin/cowbuilder --execute --configfile %s '
               '--distribution %s --basepath %s -- %s' % (pbuilderrc_path,
                                                          codename,
                                                          basepath,
                                                          batch_filepath))
    subprocess.call(shlex.split(command))


def build_package(basepath, pbuilderrc_path, dsc_path):
    """ build package for backport """
    command = ('sudo /usr/sbin/cowbuilder --build --configfile %s '
               '--basepath %s %s' % (pbuilderrc_path,
                                     basepath,
                                     dsc_path))
    return subprocess.call(shlex.split(command))


def git_build_package(codename, result_dirpath):
    ''' invoke git-buildpackage
    Must prepare ~/.gbp.conf as folloing.
    ---
    git-pbuilder-options = '--debbuildopts "-sa"'
    ---
    '''
    command = ('sudo /usr/bin/git-buildpackage --git-ignore-branch '
               '--git-pbuilder --git-export-dir=%s --git-ignore-new '
               '--git-dist=%s' % (result_dirpath, codename))
    return subprocess.call(shlex.split(command))


def change_owner(result_dirpath):
    username = pwd.getpwuid(os.geteuid()).pw_name
    command = 'sudo chown -R %s: %s' % (username, result_dirpath)
    return subprocess.call(shlex.split(command))


def update_pbuilder(basetgz):
    """ update pbuilder image """
    command = 'sudo /usr/sbin/pbuilder --update --basetgz %s' % basetgz
    return subprocess.call(shlex.split(command))


def exec_piuparts(codename, basetgz, changes_path,
                  flavor='debian', mirror=None, no_upgrade_test=False):
    """ execute piuparts for install/uninstall test """
    command_param = ['sudo', '/usr/sbin/piuparts', '-d', codename,
                     '-D', flavor, '--basetgz', basetgz, changes_path]
    if mirror:
        # for ubuntu
        command_param.insert(2, mirror)
        command_param.insert(2, '-m')
    if no_upgrade_test:
        # workaround of backport firstly
        command_param.insert(2, '--no-upgrade-test')
    return subprocess.call(command_param)


def copy_files(filename_list, src_dirpath, dest_dirpath):
    """ copy files to specified destination directory
    :param filename_list: expected Files in .changes file and .changes file
    :param src_dirpath: expected diretory of generated source package
    :param dest_dirpath: expected directory of signing key files
    """
    [shutil.copy(os.path.join(src_dirpath, filename), dest_dirpath)
     for filename in filename_list]
    return 0


def find_dsc_name(dir_path):
    """ find .dsc file name """
    return os.path.basename(glob('%s/*.dsc' % dir_path)[0])


def upload_files(codename, changes_path, reprepro_passphrase):
    """ uploading to local archive incoming directory of reprepro """
    os.environ['LANG'] = 'C'
    command = "dput %s %s" % (codename, changes_path)
    exp = pexpect.spawn(command, maxread=4000)
    exp.logfile_read = sys.stdout
    exp.expect('Please enter passphrase:')
    exp.sendline(reprepro_passphrase)
    exp.expect('Please enter passphrase:')
    exp.sendline(reprepro_passphrase)
    exp.expect(pexpect.EOF)
    exp.close()
    return exp.status


def main():
    """
    [backport with pre-build]
    1. wget .dsc file to "workspace". (as ${WORKSPACE} or current dir)
    2. parse .dsc file, generate batch_script.sh of cowbuilder for pre-build
    3. launch cowbuilder
    3-1. installing packages
    3-2. dget source package and extract it
    3-3. rebuild source package
    3-4. copy the rebuilding results to "temp/" (as BINDMOUNT of .pbuilderrc)
    4. clean build from rebuilten source package existed at "temp/".
       the building results genrate to "unsigned_results/". (as BUILDRESULT)
    5. GPG sign .dsc and .changes files in "unsigned_results/".
    6. move the source packages from "unsigned_results/" to "signed_results/".
    7. push signed source package to "incoming" with dput.

    [backport]
    1. dget source package (as ${WORKSPACE} of or current dir)
    2. clean build from downloaded source package existed at "workspace".
       the building results genrate to "unsigned_results/". (as BUILDRESULT)
    3. GPG sign .dsc and .changes files in "unsigned_results/".
    4. move the source packages from "unsigned_results/" to "signed_results/".
    5. push signed source package to "incoming" with dput.

    [build original]
    1. clean build from Git repository with git-buildpackage,
       and export results to unsigned_results/"
    2. GPG sign .dsc and .changes files in "unsigned_results/".
    3. move the source packages from "unsigned_results/" to "signed_results/".
    4. push signed source package to "incoming" with dput.
    """
    args = parse_options()

    work_dirpath = set_work_dirpath()
    temp_dirpath = os.path.join(work_dirpath, 'temp')
    unsigned_results_dirpath = os.path.join(work_dirpath, 'unsigned_results')
    signed_results_dirpath = os.path.join(work_dirpath, 'signed_results')

    make_directories([temp_dirpath,
                      unsigned_results_dirpath,
                      signed_results_dirpath])

    basepath = get_basepath(args.codename)
    basetgz = get_basetgz(args.codename)

    # update cowbuilder image tree
    print('### update cowbuilder ###')
    if update_cowbuilder(basepath) != 0:
        sys.exit(1)

    # generate .pbuilderrc
    pbuilderrc_filepath = prepare_pbuilderrc(work_dirpath, args.codename)

    if args.subparser_name == 'backport':
        # for backporting
        dsc_name = os.path.basename(args.url)
        dsc_filepath = os.path.join(work_dirpath, dsc_name)

        # retrieve source package
        print('### retreive source package ###')
        retrieve_source_package(args.url)

        if args.prebuild:
            print('### extract source package ###')
            param_dict = extract_source_package(
                os.path.join(work_dirpath, dsc_name))
            param_dict = append_param(param_dict, args)

            # temp_dirpath is for mount point with BINDMOUNTS
            param_dict['temp_dirpath'] = temp_dirpath

            batch_content = generate_batch_script(param_dict)
            batch_filepath = os.path.join(work_dirpath,
                                          param_dict.get('batch_name'))
            with open(batch_filepath, 'w') as batch_file:
                batch_file.write(batch_content)

            print('### pre build package ###')
            pre_build_package(pbuilderrc_filepath,
                              args.codename,
                              basepath,
                              batch_filepath)
            dsc_filepath = os.path.join(temp_dirpath, dsc_name)

        # build package for backport
        print('### build package with cowbuilder ###')
        if build_package(basepath, pbuilderrc_filepath, dsc_filepath) != 0:
            sys.exit(1)

    elif args.subparser_name == 'original':
        # build package with git-buildpackage
        print('### build package with git-buildpackage ###')
        if git_build_package(args.codename, unsigned_results_dirpath) != 0:
            sys.exit(1)

        # for original package
        dsc_name = find_dsc_name(unsigned_results_dirpath)

    arch = get_architecture()
    changes_name = get_changes_name(dsc_name, arch)

    # update pbuilder basetgz image
    print('### update pbuilder image ###')
    if update_pbuilder(basetgz) != 0:
        sys.exit(1)

    # install and uninstall test
    unsigned_changes_filepath = os.path.join(unsigned_results_dirpath,
                                             changes_name)

    print('### install and uninstall test ###')
    if exec_piuparts(args.codename,
                     basetgz,
                     unsigned_changes_filepath,
                     flavor=args.flavor,
                     mirror=args.mirror,
                     no_upgrade_test=args.no_upgrade_test) != 0:
        sys.exit(1)

    # change owner
    if change_owner(unsigned_results_dirpath) != 0:
        sys.exit(1)

    # debsign
    print('### debsign ###')
    changes_path = os.path.join(unsigned_results_dirpath, changes_name)
    if args.without_lintian:
        if debsign.debsign_process(changes_path,
                                   passphrase=args.passphrase,
                                   lintian=False) is False:
            sys.exit(1)
    else:
        if debsign.debsign_process(changes_path,
                                   passphrase=args.passphrase) is False:
            sys.exit(1)

    # move singed source packages
    # file list format is md5sum, size, section, priority, name
    files_list = parse_changes(unsigned_changes_filepath)

    filename_list = [_file.get('name') for _file in files_list]
    filename_list.append(changes_name)
    if copy_files(filename_list,
                  unsigned_results_dirpath,
                  signed_results_dirpath) != 0:
        sys.exit(1)

    # upload incoming directory
    print('### upload files ###')
    if upload_files(args.codename,
                    os.path.join(signed_results_dirpath, changes_name),
                    args.reprepro_passphrase) != 0:
        sys.exit(1)


if __name__ == '__main__':
    main()