Windows (TaskCluster Worker) version of https://dxr.mozilla.org/mozilla-central/source/testing/docker/desktop-build/bin/checkout-sources.sh and https://dxr.mozilla.org/mozilla-central/source/testing/taskcluster/scripts/builder/build-linux.sh

build-windows.cmd

:: Taskcluster friendly wrapper for performing fx desktop builds via mozharness.

:: Inputs, with defaults

::if not defined MOZHARNESS_SCRIPT           set MOZHARNESS_SCRIPT=mozharness\scripts\fx_desktop_build.py
::if not defined MOZHARNESS_CONFIG           set MOZHARNESS_CONFIG=builds\releng_base_windows_64_builds.py
if not defined TOOLTOOL_CACHE              set TOOLTOOL_CACHE=%SystemDrive%\home\worker\tooltool-cache

if not defined NEED_XVFB                   set NEED_XVFB=false

::if not defined MH_CUSTOM_BUILD_VARIANT_CFG set MH_CUSTOM_BUILD_VARIANT_CFG=
if not defined MH_BRANCH                   set MH_BRANCH=mozilla-central
if not defined MH_BUILD_POOL               set MH_BUILD_POOL=staging

if not defined WORKSPACE             set WORKSPACE=%SystemDrive%\home\worker\workspace

:: some linux variants, e.g. b2gdroid, require gaia
if not defined CHECKOUT_GAIA               set CHECKOUT_GAIA=false

set MOZ_CRASHREPORTER_NO_REPORT=1
set MOZ_OBJDIR=obj-firefox
set MOZ_SYMBOLS_EXTRA_BUILDID=linux64
set TINDERBOX_OUTPUT=1

:: use "simple" package names so that they can be hard-coded in the task's
:: extras.locations
set MOZ_SIMPLE_PACKAGE_NAME=target

:: Ensure that in tree libraries can be found
set LIBRARY_PATH=%LIBRARY_PATH%;$WORKSPACE\src\obj-firefox;$WORKSPACE\src\gcc\lib64

:: test required parameters are supplied
if not defined MOZHARNESS_SCRIPT exit /b 1
if not defined MOZHARNESS_CONFIG exit /b 1
if "%NEED_XVFB%" == "true" exit /b 1

:: set up mozharness configuration, via command line, env, etc.

set debug_flag=
if defined DEBUG set debug_flag=--debug

set custom_build_variant_cfg_flag=
:: if defined MH_CUSTOM_BUILD_VARIANT_CFG set custom_build_variant_cfg_flag=--custom-build-variant-cfg=%MH_CUSTOM_BUILD_VARIANT_CFG%

::if "%CHECKOUT_GAIA%" == "true" (
::  echo todo: pull gaia
  ::if exist %WORKSPACE%\build\tools rmdir %WORKSPACE%\build\tools /s /q || echo ERROR && exit /b
  ::mkdir %WORKSPACE%\build\tools || echo ERROR && exit /b
  ::hg clone -U %TOOLS_BASE_REPOSITORY% %WORKSPACE%\build\tools || echo ERROR && exit /b
  ::hg pull -u -R %WORKSPACE%\build\tools --rev %TOOLS_HEAD_REV% %TOOLS_HEAD_REPOSITORY% || echo ERROR && exit /b
  ::hg update -R %WORKSPACE%\build\tools --rev %TOOLS_HEAD_REV% || echo ERROR && exit /b
::)

:: $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the
:: cache.  However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be
:: entirely effective.
:: export TOOLTOOL_CACHE

:: support multiple, space delimited, config files
set config_cmds=
for %%c in (%MOZHARNESS_CONFIG%) do set config_cmds=%config_cmds% --config %%c

:: Mozharness would ordinarily do the checkouts itself, but they are disabled
:: here (--no-checkout-sources, --no-clone-tools) as the checkout is performed above.
python2.7 %WORKSPACE%\build\src\testing\%MOZHARNESS_SCRIPT%^
 %config_cmds%^
 %debug_flag%^
 %custom_build_variant_cfg_flag%^
 --disable-mock^
 --no-setup-mock^
 --no-checkout-sources^
 --no-clone-tools^
 --no-clobber^
 --no-update^
 --no-upload-files^
 --no-sendchange^
 --log-level=debug^
 --work-dir=%WORKSPACE%\build^
 --no-action=generate-build-stats^
 --branch=%MH_BRANCH%^
 --build-pool=%MH_BUILD_POOL%

checkout-sources.cmd

:: Inputs, with defaults

:: mozharness builds use two repositories: gecko (source)
:: and build-tools (miscellaneous) for each, specify *_REPOSITORY.  If the
:: revision is not in the standard repo for the codebase, specify *_BASE_REPO as
:: the canonical repo to clone and *_HEAD_REPO as the repo containing the
:: desired revision.  For Mercurial clones, only *_HEAD_REV is required; for Git
:: clones, specify the branch name to fetch as *_HEAD_REF and the desired sha1
:: as *_HEAD_REV.

:: if the worker type contains a hg repo cache, and BASE_REPOSITORY is not set, use the cache.
if not defined GECKO_BASE_REPOSITORY if exist %SystemDrive%\builds\hg-shared\mozilla-central\.hg\ set GECKO_BASE_REPOSITORY=C:\builds\hg-shared\mozilla-central
if not defined TOOLS_BASE_REPOSITORY if exist %SystemDrive%\builds\hg-shared\build\tools\.hg\     set TOOLS_BASE_REPOSITORY=C:\builds\hg-shared\build\tools

if not defined GECKO_REPOSITORY      set GECKO_REPOSITORY=https://hg.mozilla.org/mozilla-central
if not defined GECKO_BASE_REPOSITORY set GECKO_BASE_REPOSITORY=%GECKO_REPOSITORY%
if not defined GECKO_HEAD_REPOSITORY set GECKO_HEAD_REPOSITORY=%GECKO_REPOSITORY%
if not defined GECKO_HEAD_REV        set GECKO_HEAD_REV=default
if not defined GECKO_HEAD_REF        set GECKO_HEAD_REF=%GECKO_HEAD_REV%

if not defined TOOLS_REPOSITORY      set TOOLS_REPOSITORY=https://hg.mozilla.org/build/tools
if not defined TOOLS_BASE_REPOSITORY set TOOLS_BASE_REPOSITORY=%TOOLS_REPOSITORY%
if not defined TOOLS_HEAD_REPOSITORY set TOOLS_HEAD_REPOSITORY=%TOOLS_REPOSITORY%
if not defined TOOLS_HEAD_REV        set TOOLS_HEAD_REV=default
if not defined TOOLS_HEAD_REF        set TOOLS_HEAD_REF=%TOOLS_HEAD_REV%
if not defined TOOLS_DISABLE         set TOOLS_DISABLE=false

if not defined WORKSPACE             set WORKSPACE=%SystemDrive%\home\worker\workspace

if not "%TOOLS_DISABLE%" == "true" (
  if exist %WORKSPACE%\build\tools rmdir %WORKSPACE%\build\tools /s /q || echo ERROR && exit /b
  mkdir %WORKSPACE%\build\tools || echo ERROR && exit /b
  hg clone -U %TOOLS_BASE_REPOSITORY% %WORKSPACE%\build\tools || echo ERROR && exit /b
  hg pull -u -R %WORKSPACE%\build\tools --rev %TOOLS_HEAD_REV% %TOOLS_HEAD_REPOSITORY% || echo ERROR && exit /b
  hg update -R %WORKSPACE%\build\tools --rev %TOOLS_HEAD_REV% || echo ERROR && exit /b
)

for %%r in (%EXTRA_CHECKOUT_REPOSITORIES%) do (
  if exist "!%%r_DEST_DIR!" rmdir "!%%r_DEST_DIR!" /s /q || echo ERROR && exit /b
  mkdir "!%%r_DEST_DIR!" || echo ERROR && exit /b
  hg clone -U "!%%r_BASE_REPOSITORY!" "!%%r_DEST_DIR!" || echo ERROR && exit /b
  hg pull -u -R "!%%r_DEST_DIR!" --rev "!%%r_HEAD_REV!" "!%%r_HEAD_REPOSITORY!" || echo ERROR && exit /b
  hg update -R "!%%r_DEST_DIR!" --rev "!%%r_HEAD_REV!" || echo ERROR && exit /b
)

set GECKO_DIR=%WORKSPACE%\build\src
if exist %GECKO_DIR% rmdir %GECKO_DIR% /s /q || echo ERROR && exit /b
mkdir %GECKO_DIR% || echo ERROR && exit /b
hg clone -U %GECKO_BASE_REPOSITORY% %GECKO_DIR% || echo ERROR && exit /b
hg pull -u -R %GECKO_DIR% --rev %GECKO_HEAD_REV% %GECKO_HEAD_REPOSITORY% || echo ERROR && exit /b
hg update -R %GECKO_DIR% --rev %GECKO_HEAD_REV% || echo ERROR && exit /b

update-workertype.sh

#!/bin/bash -e

# TODO: submit a task after updating worker type

# usage: 
# TASKCLUSTER_CLIENT_ID=tc-login TASKCLUSTER_ACCESS_TOKEN=access-token TASKCLUSTER_CERTIFICATE='{}' ./update-workertype.sh win2012-vs2015

: ${TASKCLUSTER_CLIENT_ID:?"TASKCLUSTER_CLIENT_ID must have a value"}
: ${TASKCLUSTER_ACCESS_TOKEN:?"TASKCLUSTER_ACCESS_TOKEN must have a value"}
#: ${TASKCLUSTER_CERTIFICATE:?"TASKCLUSTER_CERTIFICATE must have a value"}

aws_region=${aws_region:='us-west-2'}
aws_instance_type=${aws_instance_type:='c3.2xlarge'}
#aws_instance_hdd_size=${aws_instance_hdd_size:='160'}
aws_base_ami_search_profile_win2008=${aws_base_ami_search_profile_win2008:='releng-grenade'}
aws_base_ami_search_term_win2008=${aws_base_ami_search_term_win2008:='spot-y-2008-2016-*'}
aws_base_ami_search_term_win2012=${aws_base_ami_search_term_win2012:='Windows_Server-2012-R2_RTM-English-64Bit-Base*'}
aws_tc_account_id=${aws_tc_account_id:='692406183521'}


if [ "${#}" -lt 1 ]; then
  echo "please provide a worker type, e.g. ./update-workertype.sh win2012-vs2015" >&2
  exit 64
fi
tc_worker_type="${1}"

if [ ! -d "$(dirname "${0}")/${tc_worker_type}" ]; then
  echo "ERROR: no directory for worker type: '$(dirname "${0}")/${tc_worker_type}'"
  exit 65
fi

aws_key_name="mozilla-taskcluster-worker-${tc_worker_type}"
if [ ! -f ~/.ssh/${aws_key_name} ]; then
  if aws ec2 describe-key-pairs --profile taskcluster --region ${aws_region} --key-names ${aws_key_name}; then
    echo "ERROR: key '${aws_key_name}' exists in AWS but not locally. try again when '~/.ssh/${aws_key_name}' exists."
    exit 66
  else
    ssh-keygen -t rsa -b 4096 -N "" -C "[email protected]" -f ~/.ssh/${aws_key_name}
    ssh-add ~/.ssh/${aws_key_name}
  fi
fi
if aws ec2 describe-key-pairs --profile taskcluster --region ${aws_region} --key-names ${aws_key_name}; then
  echo "$(date -Iseconds): key: ${aws_key_name} exists locally and in AWS"
else
  if [ `aws ec2 import-key-pair --profile taskcluster --region ${aws_region} --key-name ${aws_key_name} --public-key-material "$(cat ~/.ssh/${aws_key_name}.pub)"` ]; then
    echo "$(date -Iseconds): key: ~/.ssh/${aws_key_name}.pub imported to AWS (region ${aws_region})"
  fi
fi

cd "$(dirname "${0}")/${tc_worker_type}"

#go get github.com/taskcluster/generic-worker/update-worker-type
#go install github.com/taskcluster/generic-worker/update-worker-type

go get github.com/taskcluster/slugid-go/slug
tc_slug_id=$("${GOPATH}/bin/slug")
echo "$(date -Iseconds): random slugid: ${tc_slug_id} generated for aws client token"

case "${tc_worker_type}" in
  win7-32)
    aws_base_ami_id='ami-13669273'
    echo "$(date -Iseconds): current base ami for: ${tc_worker_type}, in: ${aws_region}, is: ${aws_base_ami_id}"
    ;;
  win10-64)
    aws_base_ami_id='ami-6a7c880a' # todo: fix ec2config userdata trigger and recapture ami
    echo "$(date -Iseconds): current base ami for: ${tc_worker_type}, in: ${aws_region}, is: ${aws_base_ami_id}"
    ;;
  win2008*)
    # determine latest windows 2012 r2 ami
    aws_base_ami_id="$(aws ec2 describe-images --profile ${aws_base_ami_search_profile_win2008} --region ${aws_region} --owners self --filters "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term_win2008}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)"
    aws ec2 modify-image-attribute --profile ${aws_base_ami_search_profile_win2008} --region ${aws_region} --image-id ${aws_base_ami_id} --launch-permission "{\"Add\":[{\"UserId\":\"${aws_tc_account_id}\"}]}"
    echo "$(date -Iseconds): latest base ami for: ${aws_base_ami_search_term_win2008}, in region: ${aws_region}, in profile: ${aws_base_ami_search_profile_win2008}, is: ${aws_base_ami_id}"
    ;;
  win2012*)
    # determine latest windows 2012 r2 ami
    aws_base_ami_id="$(aws ec2 describe-images --profile taskcluster --region ${aws_region} --owners amazon --filters "Name=platform,Values=windows" "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term_win2012}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)"
    echo "$(date -Iseconds): latest base ami for: ${aws_base_ami_search_term_win2012}, in: ${aws_region}, is: ${aws_base_ami_id}"
    ;;
  *)
    echo "ERROR: unknown worker type: '${tc_worker_type}'"
    exit 67
    ;;
esac

# previous instances
aws_previous_instance_ids="$(aws ec2 describe-instances --profile taskcluster --region ${aws_region} --filters Name=tag-key,Values=WorkerType "Name=tag-value,Values=${tc_worker_type}" --query 'Reservations[*].Instances[*].InstanceId' --output text)"
if [ -n "${aws_previous_instance_ids}" ]; then
  echo "$(date -Iseconds): previous instances of WorkerType: ${tc_worker_type}, include: ${aws_previous_instance_ids}"
else
  echo "$(date -Iseconds): no previous instances of WorkerType: ${tc_worker_type}"
fi

# previous snapshot
aws_previous_snapshot_id="$(aws ec2 describe-images --profile taskcluster --region ${aws_region} --owners self amazon --filters "Name=name,Values=${tc_worker_type} version*" --query 'Images[*].BlockDeviceMappings[*].Ebs.SnapshotId' --output text)"
if [ -n "${aws_previous_snapshot_id}" ]; then
  echo "$(date -Iseconds): previous snapshot: ${aws_previous_snapshot_id}"
else
  echo "$(date -Iseconds): no previous snapshot"
fi

# previous ami
aws_previous_ami_id="$(aws ec2 describe-images --profile taskcluster --region ${aws_region} --owners self amazon --filters "Name=name,Values=${tc_worker_type} version*" --query 'Images[*].ImageId' --output text)"
if [ -n "${aws_previous_ami_id}" ]; then
  echo "$(date -Iseconds): previous ami: ${aws_previous_ami_id}"
else
  echo "$(date -Iseconds): no previous ami"
fi

# create instance, apply user-data, filter output, get instance id, tag instance, wait for shutdown
aws_instance_id="$(aws ec2 run-instances --profile taskcluster --region ${aws_region} --image-id "${aws_base_ami_id}" --key-name ${aws_key_name} --security-groups "ssh-only" "rdp-only" --user-data "$(cat userdata.ps1)" --instance-type ${aws_instance_type} --block-device-mappings DeviceName=/dev/sda1,Ebs='{VolumeSize=160,DeleteOnTermination=true,VolumeType=gp2}' --instance-initiated-shutdown-behavior stop --client-token "${tc_slug_id}" | sed -n 's/^ *"InstanceId": "\(.*\)", */\1/p')"
aws ec2 create-tags --profile taskcluster --region ${aws_region} --resources "${aws_instance_id}" --tags "Key=WorkerType,Value=${tc_worker_type}"
echo "$(date -Iseconds): instance: ${aws_instance_id} instantiated and tagged: WorkerType=${tc_worker_type}"
sleep 30 # give aws 30 seconds to start the instance
echo "$(date -Iseconds): userdata logging to: https://papertrailapp.com/groups/2488493"
aws_instance_public_ip="$(aws ec2 describe-instances --profile taskcluster --region ${aws_region} --instance-id "${aws_instance_id}" --query 'Reservations[*].Instances[*].NetworkInterfaces[*].Association.PublicIp' --output text)"
echo "$(date -Iseconds): instance public ip: ${aws_instance_public_ip}"

case "${tc_worker_type}" in
  "win7-32")
    echo "$(date -Iseconds): rdp: rdesktop -u Administrator -p '$(pass Mozilla/relops/ec2/Administrator@t-w732)' -k en-gb -g 2400x1200 -a 16 -K -r clipboard:CLIPBOARD ${aws_instance_public_ip}"
    ;;
  "win10-64")
    echo "$(date -Iseconds): rdp: xfreerdp /u:root /p:'$(pass Mozilla/relops/ec2/root@t-w1064)' /kbd:809 /w:2400 /h:1200 +clipboard /v:${aws_instance_public_ip}"
    ;;
  "win2008")
    echo "$(date -Iseconds): rdp: rdesktop -u root -p '$(pass Mozilla/relops/ec2/root@y-2008)' -k en-gb -g 2400x1200 -a 16 -K -r clipboard:CLIPBOARD ${aws_instance_public_ip}"
    ;;
  *)
    sleep 210 # give ec2config a further 3 and a half minutes to boot and generate a password for the Administrator account
    until [ -n "${aws_instance_password}" ];
    do
      aws_instance_password="$(aws ec2 get-password-data --profile taskcluster --region ${aws_region} --instance-id "${aws_instance_id}" --priv-launch-key ~/.ssh/${aws_key_name} --output text --query PasswordData)"
      sleep 30
    done
    echo "$(date -Iseconds): instance credentials: Administrator : ${aws_instance_password}"
    echo "$(date -Iseconds): rdp: rdesktop -u Administrator -p '${aws_instance_password}' -k en-gb -g 2400x1200 -a 16 -K -r clipboard:CLIPBOARD ${aws_instance_public_ip}"
    ;;
esac

# poll for a stopped state
until `aws ec2 wait instance-stopped --profile taskcluster --region ${aws_region} --instance-ids "${aws_instance_id}" >/dev/null 2>&1`;
do
  echo "$(date -Iseconds): waiting for instance to shut down"
done

aws_ami_id=`aws ec2 create-image --profile taskcluster --region ${aws_region} --instance-id ${aws_instance_id} --name "${tc_worker_type} version ${tc_slug_id}" --description "Firefox desktop builds for Windows - TaskCluster ${tc_worker_type} worker - version ${tc_slug_id}" | sed -n 's/^ *"ImageId": *"\(.*\)" *$/\1/p'`
echo "$(date -Iseconds): ami: ${aws_ami_id} creation in progress: https://${aws_region}.console.aws.amazon.com/ec2/v2/home?region=${aws_region}#Images:visibility=owned-by-me;search=${aws_ami_id};sort=desc:platform"
sleep 30
until `aws ec2 wait image-available --profile taskcluster --region ${aws_region} --image-ids "${aws_ami_id}" >/dev/null 2>&1`;
do
  echo "$(date -Iseconds): waiting for ami availability"
done

if "${GOPATH}/bin/update-worker-type" "${aws_ami_id}" "${tc_worker_type}"; then
  echo "$(date -Iseconds): worker type updated: https://tools.taskcluster.net/aws-provisioner/#${tc_worker_type}/edit"
else
  echo "$(date -Iseconds): worker type update failed: https://tools.taskcluster.net/aws-provisioner/#${tc_worker_type}/edit"
fi

{
  echo "Instance Id: ${aws_instance_id}"
  echo "Public IP:   ${aws_instance_public_ip}"
  echo "Password:    ${aws_instance_password}"
  echo "AMI Id:      ${aws_ami_id}"
} > latest.txt

# terminate previous instances
if [ -n "${aws_previous_instance_ids}" ]; then
  echo "$(date -Iseconds): terminating previous instances: ${aws_previous_instance_ids}"
  aws ec2 terminate-instances --profile taskcluster --region ${aws_region} --instance-ids ${aws_previous_instance_ids} #>/dev/null 2>&1
else
  echo "$(date -Iseconds): no previous instances to terminate"
fi

# deregister old ami
if [ -n "${aws_previous_ami_id}" ]; then
  for ami_id in ${aws_previous_ami_id}
  do
    echo "$(date -Iseconds): deregistering previous ami: ${ami_id}"
    aws ec2 deregister-image --profile taskcluster --region ${aws_region} --image-id "${ami_id}"
  done
else
  echo "$(date -Iseconds): no previous amis to deregister"
fi
sleep 30 # give the ami(s) a little time to deregister before deleting snapshots
# delete old snapshot
if [ -n "${aws_previous_snapshot_id}" ]; then
  for snapshot_id in ${aws_previous_snapshot_id}
  do
    echo "$(date -Iseconds): deleting previous snapshot: ${snapshot_id}"
    aws ec2 delete-snapshot --profile taskcluster --region ${aws_region} --snapshot-id "${snapshot_id}"
  done
else
  echo "$(date -Iseconds): no previous snapshot to delete"
fi