Debian packaging continuous integration

00-packages_builder.rb

# Chef resources describing how to set up package repository server,
# simplified fromactual cookbook (not open sourced yet). Sets up apt
# repository in /srv/apt directory, with system user "apt-repo". Packages
# are GPG-signed to prevent apt-get from complaining on every install.
# 
# Directory /srv/apt should be reachable to clients via http or other
# means. This is left as an exercise for the reader.
#
# For extra explanations, see:
# http://joseph.ruscio.org/blog/2010/08/19/setting-up-an-apt-repository/
# http://www.roethof.net/techblog/setting-up-an-apt-repository-with-reprepro/
# http://blog.mycrot.ch/2011/04/26/creating-your-own-signed-apt-repository-and-debian-packages/

package "reprepro"              # manages apt repository
package "gnupg"                 # signs packages / indexes
package "build-essential"       # glibc, gcc, and company
package "dpkg-dev"              # basic debian packaging tools - just in case
package "fakeroot"              # let us pretend we can chown stuff to root
gem_package "fpm"               # the actual tool that packages stuff

# System user & group
group "apt-repo"
user "apt-repo" do
  system true
  group "apt-repo"
  home "/srv/apt"
end

# Home & config directory
%w(/srv/apt /srv/apt/conf).each do |dir_path|
  directory dir_path do
    owner "apt-repo"
    group "apt-repo"
    mode "0755"
  end
end

# Distributions file - see reprepro docs for details
template "/srv/apt/conf/distributions" do
  owner "apt-repo"
  group "apt-repo"
  mode "0644"
  variables :distributions => node.apt_repository_distributions
end

# Specification for non-interactive GPG key generation
file "#{Chef::Config[:file_cache_path]}/apt-key-spec.txt" do
  content <<EOF
Key-Type: 1
Key-Length: 2048
Name-Real: #{node[:apt_repository][:key_real_name]}
Name-Email: #{node[:apt_repository][:key_email]}
Expire-Date: 0   
EOF
end

# Generate GPG key (non-interactively with --batch)
execute "generate-apt-gpg-key" do
  command "gpg --gen-key --batch < #{Chef::Config[:file_cache_path]}/apt-key-spec.txt"
  user 'apt-repo'
  group 'apt-repo'
  environment "HOME" => '/srv/apt'
  not_if { File.exist?('/srv/apt/.gnupg') }
end

# Remember the public key as a Chef attribute
ruby_block "save-apt-gpg-key" do
  block do
    node[:apt_repository][:public_gpg_key] = `sudo -u apt-repo -H gpg --export --armor`
  end
end

# Helper script for simplicity
cookbook_file "/srv/apt/add_package.sh" do
  owner "apt-repo"
  group "apt-repo"
  mode "0755"
end

00b-add_package.sh

#!/bin/sh

# Usage: /srv/apt/add_package.sh package.deb [package2.deb [package3.deb [...]]]

DISTRO=${DISTRO:-squeeze}

set -e -x

while [ -f "$1" ] ; do
    sudo -u apt-repo reprepro -Vb /srv/apt includedeb $DISTRO "$1"
    shift
done

01_client.rb

# Chef resources to set up repository client.

package "lsb-release"

# Save public GPG key from Chef attribute to file
cached_keyfile = "#{Chef::Config[:file_cache_path]}/#{server.name}.apt-key"
file cached_keyfile do
  content server[:apt_repository][:public_gpg_key]
end

# Add key as trusted
execute "install-apt-key #{server.name}" do
  command "apt-key add #{cached_keyfile}"
end

# Figure out full repository URL
url = "http://"
url << "#{server[:apt_repository][:http_username]}:#{server[:apt_repository][:http_password]}@" if server[:apt_repository][:http_username]
url << "#{server[:apt_repository][:http_domain]}/"

# Add repository list
file "/etc/apt/sources.list.d/chef_apt_repository.list" do
  content "deb #{url} #{server[:apt_repository][:codename]} #{server[:apt_repository][:components]}\n"
  owner 'root'
  group 'sysadmin'
  mode '0640'
  notifies :run, 'execute[apt-get update]', :immediately
end

02_packages_Gemfile.rb

# -*- ruby -*-
# Ruby gems used to build packages

source "http://rubygems.org"

gem "rake"                      # Build tool
gem "evoker", ">= 0.0.9"        # Add-on to Rake for downloading upstream stuff
gem "fpm"                       # Effing Package Management
gem "escape"                    # Shell escapes for scripting

gem "vagrant", '~> 1.0.3', :group => :development
# VirtualBox command-line tool for developing packages on a clean VM

03_selenium-server_Rakefile.rb

# Each package lives in its own subdirectory of the packages repo.
# Each package directory has its own Rakefile.
# Here's the structure of the Rakefile:

# First, some general constants used by magic later on.
PKG_VERSION = '2.21.0'
PKG_ITERATION = 2
PKG_ARCHITECTURE = 'all'
PKG_DEPS = ['sun-java6-jre', 'runit', 'xvfb', 'gnowsis-firefox (>= 11.0)']

# More specific constants, used by code below
CHECKSUM = '34983fbf870cefeab892786e3d4f9169ed7e9e17e6937b5fba5e2133603ca03a'
URL="http://selenium.googlecode.com/files/selenium-server-standalone-#{PKG_VERSION}.jar"

# Include the magic
load '../packaging.rake'
require 'evoker'
require 'evoker/local_cache'
include Evoker

# Download file to package
dl = cached_wget(URL,
  :output_file => 'root/opt/selenium-server/selenium-server-standalone.jar',
  :checksum => CHECKSUM)

# Call 'fpm' script via utility method defined in packaging.rake to build actual package

file PKG_FILE_NAME => dl do
  fpm '-s dir --after-install after-install -C root opt/selenium-server'
end

05_dwim_all.sh

# This is the Buildbot step to rebuild all updated packages and add them to
# apt repository. It calls apt_dwim task for every subdirectory with a Rakefile.
#
# DWIM stands for "Do What I Mean": if the package to be built is already in the
# apt repository, this task does nothing; if the package is not there, it does a clean
# rebuild and adds it to the apt repository.

set -e
for rf in */Rakefile ; do
    d=`dirname $rf`
    cd $d
    echo "*** $d"
    fakeroot bundle exec rake apt_dwim
    cd ..
done

06_packaging.rake

# -*- ruby -*-

# The magic happens here.
# This file is included from all the Rakefiles after defining basic constants

require 'rake/clean'

APT_ROOT = ENV['APT_ROOT'] || '/srv/apt'

# Set a constant if it's not defined yet
def _setdefault(const, value)
  Object.const_set(const, value) unless Object.const_defined?(const)
end

# Try to figure out default values for constants that haven't been yet defined.
_setdefault(:PKG_NAME, File.basename(Dir.pwd))
_setdefault(:PKG_VERSION, '0.unspecified')
_setdefault(:PKG_ITERATION, 1)
_setdefault(:PKG_ARCHITECTURE, 'amd64')
_setdefault(:PKG_DISTROS, %w(squeeze))
_setdefault(:PKG_DEPS, [])
_setdefault(:PKG_BUILD_DEPS, [])

_setdefault(:PKG_FILE_NAME, "gnowsis-#{PKG_NAME}_#{PKG_VERSION}-gnowsis#{PKG_ITERATION}_#{PKG_ARCHITECTURE}.deb")
_setdefault(:PKG_FILES, [PKG_FILE_NAME]) # This will be defined earlier if the Rakefile
                                         # builds multiple .deb files

CACHE_PATH = '../cache' # Download cache is shared between packages

CLEAN.include(*PKG_FILES)

# Call fpm tool, filling in the defaults
def fpm(*args)
  require 'escape'
  cmd = "bundle exec fpm -t deb -n gnowsis-#{PKG_NAME} "
  cmd << "-v #{PKG_VERSION}  --iteration gnowsis#{PKG_ITERATION} "
  cmd << "-a #{PKG_ARCHITECTURE} -m [email protected] --license proprietary "
  PKG_DEPS.each { |d| cmd << "-d #{Escape.shell_single_word(d)} " }
  cmd << args.join(' ')
  sh cmd
end

### Check build dependencies

# Return true if package is installed in the system.
def pkg_installed?(package)
  system "dpkg-query -s #{package} > /dev/null 2>&1"
  return $?.exitstatus.zero?
end

missing = PKG_BUILD_DEPS.select { |pkg| !pkg_installed?(pkg) }
unless missing.empty? || ENV['IGNORE_MISSING'] # Allow user to ignore missing build deps
  puts "Missing packages:\n - #{missing.join("\n - ")}"

  if ENV['INSTALL_DEPS'] # Allow user to let rake automatically install missing build deps
    sh "sudo apt-get install --yes #{missing.join(' ')}"
  else
    puts "Install with: sudo apt-get install #{missing.join(' ')}"
    raise "Cannot proceed."
  end
end

### Apt repository stuff

# True if package_filename package is already in the apt repo
def already_in_apt?(package_filename)
  !Dir[
    File.join(APT_ROOT, 'pool', '*', '*', '*', File.basename(package_filename))
  ].empty?
end

desc "Build package and add it to apt"
task :add_to_apt => PKG_FILES do
  PKG_FILES.each do |pkg|
    sh "#{APT_ROOT}/add_package.sh #{pkg}"
  end
end

desc "If package is not in apt repository, do clean rebuild, and add to apt."
if PKG_FILES.all? { |pkg| already_in_apt?(pkg) } and not ENV['FORCE_REBUILD']
  task :apt_dwim do
    puts "All packages already in repo, doing nothing."
  end
else
  task :apt_dwim => [ :clean, :add_to_apt ]
end

desc "Build package"
task :default => PKG_FILES

07_Vagrantfile.rb

# -*- mode: ruby -*-
# vi: set ft=ruby :

# Configuration file for Vagrant (http://vagrantup.com/) to set up a virtual machine
# for package development.

# Figure out user data to set up git on the VM
USER_EMAIL = `git config user.email`.strip
USER_NAME = `git config user.name`.strip

# Find address of our custom apt repository to configure it automatically
# on the VM.
APT_REPOSITORY = `ssh [APT_HOST] cat /etc/apt/sources.list.d/chef_apt_repository.list`.strip
APT_KEY = `ssh [APT_HOST] sudo apt-key export apt-repo@[APT_HOST]`.strip

Vagrant::Config.run do |config|
  config.ssh.forward_agent = true

  config.vm.define "gnowsis-packages-vm" do |vm_cfg|
    # Clean Debian Lenny base box
    # http://dominique.broeglin.fr/2011/03/26/squeeze-64-vagrant-base-box.html
    vm_cfg.vm.box = "squeeze64"
    vm_cfg.vm.box_url = "http://dl.dropbox.com/u/937870/VMs/squeeze64.box"

    vm_cfg.vm.provision :shell, :inline => <<EOF
# Provisioning script, ran as root when VM is first started
set -e -x

# Configure VM as client for custom apt repository
cat > /etc/apt/sources.list.d/chef_apt_repository.list <<EOR
#{APT_REPOSITORY}
EOR

cat <<EOK | apt-key add -
#{APT_KEY}
EOK

# General upgrade
apt-get update
apt-get upgrade --yes

# Needed packages (libxml2 & libxslt are needed as gems' prereqs later on)
apt-get install --yes git build-essential libxml2-dev libxslt1-dev

# Update rubygems themselves
gem install rubygems-update
update_rubygems

# Install gem bundler (http://gembundler.com/) to automatically install
# Ruby gems from the Gemfile
gem install bundler

# Share cache path with VM's host (directory of the Vagrantfile is seen by the
# VM as /vagrant/); install build dependencies automatically.
cat > /etc/profile.d/gnowsis_packages_settings.sh <<EOS
export CACHE_PATH=/vagrant/cache
export INSTALL_DEPS=1
EOS

# Now set up non-root account which we are `vagrant ssh'-ing.
su vagrant <<EOSU
cd ~
set -e -x

git config --global user.email #{USER_EMAIL.inspect}
git config --global user.name #{USER_NAME.inspect}

# Clone host VM's repository, using .git dir as bare repo
git clone /vagrant/.git packages

cd packages

# Create "develop" branch. It will fail if develop branch exists on host machine,
# but is not merged to master.
git checkout -b develop
git push origin develop:develop

# Download Ruby gems needed to run the Rakefiles
bundle --binstubs --path bundle/
EOSU

EOF
  end
end

08_java_Rakefile.rb

# With that setup, we don't even need to use fpm to build
# packages. For example, it's possible to build traditionally packaged
# https://github.com/rraptorr/sun-java6/ Oracle Java debs, by calling
# out to dpkg-buildpackage instead of fpm.
#
# If we want to use third-party packages that don't have their apt
# repository, we can skip the "build" part altogether and just
# download the .deb files and consider them ready.

PKG_VERSION = '6.32-1'
PKG_BUILD_DEPS = %w(lib32asound2 ia32-libs unixodbc libx11-6 libxext6 libxi6 libxt6 libxtst6)

PKG_FILES = { 'ia32-sun-java6-bin' => 'amd64',
              'sun-java6-bin' => 'amd64',
              'sun-java6-fonts' => 'all',
              'sun-java6-javadb' => 'all',
              'sun-java6-jdk' => 'amd64',
              'sun-java6-jre' => 'all',
              'sun-java6-plugin' => 'amd64',
              'sun-java6-source' => 'all'
            }.map { |name, arch| "#{name}_#{PKG_VERSION}_#{arch}.deb" }

GIT_TAG = "v#{PKG_VERSION}"

# These files are downloaded automatically from S3 by packaging.rake - this part of
# that file has been left out for clarity.

S3_FILES = {
  "jdk-6u32-linux-i586.bin" => 'ab840c7de8f452b09d53ebe9477ffdb4622fa8cabbf962474d1914fde9a20f19',
  "jdk-6u32-linux-x64.bin" => '269d05b8d88e583e4e66141514d8294e636f537f55eb50962233c9e33d8f8f49'
}

load '../packaging.rake'
require 'evoker'
include Evoker

pkg_repo = git('rraptor-sun-java6',
  :url => 'git://github.com/rraptorr/sun-java6.git',
  :revision => GIT_TAG)

task :build => [:s3_files, pkg_repo] do
  S3_FILES.keys.each do |binf|
    ln_sf "../#{binf}", "#{pkg_repo}/#{binf}"
  end
  sh "cd #{pkg_repo} && dpkg-buildpackage -b -uc -us"
end

PKG_FILES.each { |deb_file| file deb_file => [ :build ] }

CLEAN << pkg_repo
CLEAN << "sun-java6_#{PKG_VERSION}_#{PKG_ARCHITECTURE}.changes"

09_jetty_Rakefile.rb

# A more involved fpm-using Rakefile

PKG_VERSION = '8.1.3.v20120416'
PKG_ITERATION = 5
PKG_ARCHITECTURE = 'all'
PKG_DEPS = %w(sun-java6-jdk runit)

CHECKSUM = 'd92baa45e100b348a8238f6199d194122719e14de23ece86821068d4bededbec'
URL="http://download.eclipse.org/jetty/#{PKG_VERSION}/dist/jetty-distribution-#{PKG_VERSION}.tar.gz"

load '../packaging.rake'
require 'evoker'
require 'evoker/local_cache'
include Evoker

dl = cached_wget(URL, :checksum => CHECKSUM)

task :install => dl do
  sh "tar -C root/opt/jetty --strip-components 1 --exclude '*/javadoc' -xzf #{dl}"
  chown_R 'root', 'root', 'root' if Process::Sys.getuid.zero? # fakeroot?
end

task :clean do
  sh "git clean -fdX root/"
end

file PKG_FILE_NAME => :install do
  cfcf = '--config-files /opt/jetty/etc/keystore'
  Dir.chdir 'root' do
    Dir['opt/jetty/etc/*.{xml,properties,conf,ini}'].each do |cf|
      cfcf << " --config-files /#{cf}"
    end
  end
  fpm "-s dir --before-install preinst.sh --after-install postinst.sh --before-remove prerm.sh -C root #{cfcf} opt/"
end