2008-userdata.ps1

<powershell>
<#
.Synopsis
  Bootstrap an AWS EC2 instance with cloud-tools (https://github.com/mozilla/build-cloud-tools)
.Description
  This script will do the following when run:
  - log output to the puppet mailing list, to enable debugging
  - check and/or correct the hostname if required
  - check and/or correct the primary dns suffix
  - check and/or correct the nxlog aggregator configuration
  - ensure only a single puppet run on the golden instance
  - handle reboots and shutdowns (eg: after hostname changes or puppet runs)
.Notes
  All parameters have sensible defaults and only need to be provided when running outside of the cloud-tools environment (eg: testing).
  If running outside of cloud tools, ensure that you provide sensible values for: hostname, domain & aggregator.
  Double angle brackets used in this script are intentional and used to escape the bracket when python pre-processing replaces single bracket tokens.
.Parameter logRecipients
  Defines the email recipient list for log reports generated by this script.
  Accepts a string, if only one recipient, or a string array, if more than one.
  Defaults to: '[email protected]', if not defined.
.Parameter smtpServer
  Defines the smtp server that will relay log messages.
  Defaults to: 'smtp.mail.scl3.mozilla.com', if not defined.
.Parameter logDir
  Defines the local directory on the target host where log files will be generated and kept.
  The directory will be created if it does not exist.
  Defaults to: 'C:\log', if not defined.
.Parameter runLog
  Defines the physical path of the log file for this script.
  Defaults to: $logDir + '\userdata-run-' + <timestamp> + '.log', if not defined.
.Parameter puppetLog
  Defines the physical path of the log file for the puppet agent run.
  Defaults to: $logDir + '\puppet-agent-run.log', if not defined.
.Parameter hostname
  Defines the hostname of the target instance.
  When run by cloud-tools, the hostname token will be replaced before the script is uploaded to the target instance.
  Defaults to: '{{hostname}}', if not defined.
.Parameter domain
  Defines the primary DNS suffix of the target instance.
  When run by cloud-tools, the domain token will be replaced before the script is uploaded to the target instance.
  Defaults to: '{{domain}}', if not defined.
.Parameter aggregator
  Defines the hostname of the target instance.
  When run by cloud-tools, the region_dns_atom token will be replaced before the script is uploaded to the target instance.
  Defaults to: 'log-aggregator.srv.releng.{{region_dns_atom}}.mozilla.com', if not defined.
.Example
  PS>.\userscript.ps1 -hostname b-2008-ec2-grenade -domain build.releng.use1.mozilla.com -aggregator log-aggregator.srv.releng.use1.mozilla.com
#>
param (
  # environment configuration
  [string[]] $logRecipients = @('[email protected]'),
  [string] $smtpServer = 'smtp.mail.scl3.mozilla.com',

  # runtime configuration
  [string] $logDir = ('{{0}}\log' -f $env:SystemDrive),
  [string] $runLog = [IO.Path]::Combine($logDir, ('userdata-run-{{0}}.log' -f [DateTime]::Now.ToString("yyyyMMdd-HHmm"))),
  [string] $puppetLog = [IO.Path]::Combine($logDir, 'puppet-agent-run.log'),

  # accept tokens from cloud-tools (change tokens to actual values if running outside cloud-tools)
  [string] $hostname = '{hostname}',
  [string] $domain = '{domain}',
  [string] $aggregator = 'log-aggregator.srv.releng.{region_dns_atom}.mozilla.com'
)
$global:logRecipients = $logRecipients
$global:smtpServer = $smtpServer
$global:runLog = $runLog
$global:puppetLog = $puppetLog
$global:hostname = $hostname
$global:domain = $domain
$global:aggregator = $aggregator

function Write-Log {{
  <#
  .Synopsis
    Logs to the userdata run log file, with timestamps.
  .Parameter message
    The body of the log message
  .Parameter severity
    The severity of the message, to enable filtering in log aggregators or reporting.
  .Parameter path
    The full path to the log file.
  #>
  param (
    [string] $message,
    [string] $severity = 'INFO',
    [string] $path = $global:runLog
  )
  Add-Content -Path $path ('{{0}} [{{1}}] {{2}}' -f [DateTime]::Now.ToString("yyyy-MM-dd HH:mm:ss"), $severity, $message)
}}

function Send-Log {{
  <#
  .Synopsis
    Mails the specified logfile to the configured recipient(s)
  .Parameter logfile
    The full path to the log file to be mailed.
  .Parameter subject
    The subject line of the message.
  .Parameter to
    The recipient(s) of the message.
  .Parameter from
    The sender of the message.
  .Parameter smtpServer
    The smtp server that relays log messages.
  #>
  param (
    [string] $logfile,
    [string] $subject,
    [string] $to = $global:logRecipients,
    [string] $from = ('{{0}}@{{1}}.{{2}}' -f $env:UserName, $global:hostname, $global:domain),
    [string] $smtpServer = $global:smtpServer
  )
  Send-MailMessage -To $to -Subject $subject -Body ([IO.File]::ReadAllText($logfile)) -SmtpServer $smtpServer -From $from
}}

function Stop-ComputerWithDelay {{
  <#
  .Synopsis
    Shuts down the computer and optionally restarts, logging a reason to the event log.
  .Parameter reason
    The reason for the shutdown or reboot.
  .Parameter delayInSeconds
    The time delay in seconds before shutting down
  .Parameter restart
    Whether or not to restart after shutdown
  #>
  param (
    [string] $reason,
    [int] $delayInSeconds = 10,
    [switch] $restart
  )
  Write-Log -message ('shutting down with reason: {{0}}' -f $reason) -severity 'INFO'
  if ($restart) {{
    $stopArgs = @('-r', '-t', $delayInSeconds, '-c', $reason, '-f', '-d', 'p:4:1')
  }} else {{
    $stopArgs = @('-s', '-t', $delayInSeconds, '-c', $reason, '-f', '-d', 'p:4:1')
  }}
  & 'shutdown' $stopArgs
}}

function Does-FileContain {{
  <#
  .Synopsis
    Determine if a file contains the specified string
  .Parameter needle
    The string to search for.
  .Parameter haystack
    The full path to the file to be checked.
  #>
  param (
    [string] $haystack,
    [string] $needle
  )
  if (((Get-Content $haystack) | % {{ $_ -Match "$needle" }}) -Contains $true) {{
    return $true
  }} else {{
    return $false
  }}
}}

function Has-PuppetRunSuccessfully {{
  <#
  .Synopsis
    Determine if a successful puppet run has completed
  #>
  if ((Test-Path $global:puppetLog) -and (Does-FileContain -haystack $global:puppetLog -needle "Puppet (notice): Finished catalog run")) {{
    return $true
  }} else {{
    return $false
  }}
}}

# stops and disables the puppet service and deletes the RunPuppet scheduled task
function Disable-PuppetService {{
  <#
  .Synopsis
    Stops and disables the puppet service and deletes the RunPuppet scheduled task
  #>
  Write-Log -message 'stopping and disabling puppet service' -severity 'INFO'
  Get-Service puppet | Stop-Service -PassThru | Set-Service -StartupType disabled

  Write-Log -message 'deleting RunPuppet scheduled task' -severity 'INFO'
  Unregister-ScheduledTask -TaskName 'RunPuppet' -Confirm:$false
}}

function Run-Puppet {{
  <#
  .Synopsis
    Runs the puppet agent
  .Description
    Runs the puppetization vbscript
    Runs the puppet agent in cli mode, logging to an output file
    Deletes the RunPuppet scheduled task
  .Parameter hostname
    The hostname of the instance, required for facter env vars.
  .Parameter domain
    The domain of the instance, required for facter env vars.
  #>
  param (
    [string] $hostname,
    [string] $domain,
    [string] $logdest = $global:puppetLog
  )
  Write-Log -message 'setting environment variables' -severity 'INFO'
  [Environment]::SetEnvironmentVariable("FACTER_domain", "$domain", "Process")
  [Environment]::SetEnvironmentVariable("FACTER_hostname", "$hostname", "Process")
  [Environment]::SetEnvironmentVariable("FACTER_fqdn", ("$hostname.$domain"), "Process")
  [Environment]::SetEnvironmentVariable("COMPUTERNAME", "$hostname", "Machine")

  Write-Log -message 'running puppetization script' -severity 'INFO'
  #todo: log and mail output from vbs script
  cscript.exe ('{{0}}\Puppetlabs\puppet\var\puppettize_TEMP.vbs' -f $env:ProgramData)
  
  Write-Log -message ('running puppet agent, logging to: {{0}}' -f $logdest) -severity 'INFO'
  $puppetArgs = @('agent', '--test', '--detailed-exitcodes', '--server', 'puppet', '--logdest', $logdest)
  & 'puppet' $puppetArgs

  Write-Log -message 'deleting RunPuppet scheduled task (again)' -severity 'INFO'
  Unregister-ScheduledTask -TaskName 'RunPuppet' -Confirm:$false
}}

function Is-HostnameSetCorrectly {{
  <#
  .Synopsis
    Determines if the hostname is correctly set
  .Parameter hostnameExpected
    The expected hostname of the instance.
  #>
  param (
    [string] $hostnameExpected
  )
  $hostnameActual = [System.Net.Dns]::GetHostName()
  if ("$hostnameExpected" -ieq "$hostnameActual") {{
    return $true
  }} else {{
    Write-Log -message ('net dns hostname: {{0}}, expected: {{1}}' -f $hostnameActual, $hostnameExpected) -severity 'DEBUG'
    Write-Log -message ('computer name env var: {{0}}, expected: {{1}}' -f $env:COMPUTERNAME, $hostnameExpected) -severity 'DEBUG'
    return $false
  }}
}}

function Set-Hostname {{
  <#
  .Synopsis
    Sets the hostname
  .Description
    - Sets the COMPUTERNAME environment variable at the machine level
    - Renames the computer
    - Adds the new hostname to the sysprep file, to prevent sysprep from reverting the hostname on reboot
  .Parameter hostname
    The required new hostname of the instance.
  #>
  param (
    [string] $hostname
  )
  [Environment]::SetEnvironmentVariable("COMPUTERNAME", "$hostname", "Machine")
  (Get-WmiObject Win32_ComputerSystem).Rename("$hostname")
  Write-Log -message ('hostname set to: {{0}}' -f $hostname) -severity 'INFO'
  $sysprepFile = ('{{0}}\Amazon\Ec2ConfigService\sysprep2008.xml' -f $env:ProgramFiles)
  [xml] $xml = Get-Content($sysprepFile)
  foreach ($settings in $xml.DocumentElement.settings) {{
    if ($settings.pass -eq "specialize") {{
      foreach ($component in $settings.component) {{
        if ($component.name -eq "Microsoft-Windows-Shell-Setup") {{
          if (-not $component.ComputerName) {{
            $computerNameElement = $xml.CreateElement("ComputerName")
            $computerNameElement.value = "$hostname"
            $component.AppendChild($computerNameElement)
            Write-Log -message ('computer name inserted to: {{0}}' -f $sysprepFile) -severity 'DEBUG'
          }} else {{
            $component.ComputerName.value = "$hostname"
            Write-Log -message ('computer name updated in: {{0}}' -f $sysprepFile) -severity 'DEBUG'
          }}
        }}
      }}
    }}
  }}
  $xml.Save($sysprepFile)
}}

function Is-DomainSetCorrectly {{
  <#
  .Synopsis
    Determines if the primary dns suffix is correctly set
  .Parameter domainExpected
    The expected primary dns suffix of the instance.
  #>
  param (
    [string] $domainExpected
  )
  $primaryDnsSuffix = (Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\" -Name "NV Domain")."NV Domain"
  if ("$domainExpected" -ieq "$primaryDnsSuffix") {{
    return $true
  }} else {{
    Write-Log -message ('nv domain registry key: {{0}}, expected: {{1}}' -f $primaryDnsSuffix, $domainExpected) -severity 'DEBUG'
    return $false
  }}
}}

function Set-Domain {{
  <#
  .Synopsis
    Set the primary DNS suffix (for FQDN)
  .Parameter domain
    The required new primary DNS suffix of the instance.
  #>
  param (
    [string] $domain
  )
  Set-ItemProperty 'HKLM:\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\' -Name 'Domain' -Value "$domain"
  Set-ItemProperty 'HKLM:\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\' -Name 'NV Domain' -Value "$domain"
  Write-Log -message ('Primary DNS suffix set to: {{0}}' -f $domain) -severity 'INFO'
}}

# determines if the log aggregator is correctly set
function Is-AggregatorConfiguredCorrectly {{
  <#
  .Synopsis
    Determines if the log aggregator is correctly set
  .Parameter aggregator
    The fqdn of the log aggregator for the current aws region.
  #>
  param (
    [string] $aggregator
  )
  $conf = ('{{0}}\nxlog\conf\nxlog_target_aggregator.conf' -f ${{env:ProgramFiles(x86)}})
  if ((Test-Path $conf) -and (Does-FileContain -haystack $conf -needle $aggregator)) {{
    return $true
  }} else {{
    return $false
  }}
}}

function Set-Aggregator {{
  <#
  .Synopsis
    Sets the fqdn of the log aggregator for the current aws region.
  .Description
    Modifies the nxlog configuration file to point to the specified log aggregator and restarts the nxlog service.
  .Parameter aggregator
    The fqdn of the log aggregator for the current aws region.
  #>
  param (
    [string] $aggregator
  )
  $conf = ('{{0}}\nxlog\conf\nxlog_target_aggregator.conf' -f ${{env:ProgramFiles(x86)}})
  if (Test-Path $conf) {{
    (Get-Content $conf) | 
      Foreach-Object {{ $_ -replace "(Host [^ ]*)", "Host $aggregator" }} | 
        Set-Content $conf
    Restart-Service nxlog
    Write-Log -message "log aggregator set to: $aggregator" -severity 'INFO'
  }}
}}

# create the log directory if it doesn't exist
New-Item -ItemType Directory -Force -Path $logDir
Disable-PuppetService
$hostnameCorrect = (Is-HostnameSetCorrectly -hostnameExpected $hostname)
$domainCorrect = (Is-DomainSetCorrectly -domainExpected $domain)
if ($hostnameCorrect -and $domainCorrect) {{
  Write-Log -message ('correct hostname ({{0}}) detected' -f $hostname) -severity 'INFO'
  Write-Log -message ('correct domain ({{0}}) detected' -f $domain) -severity 'INFO'
  if (Has-PuppetRunSuccessfully) {{
    Write-Log -message 'skipping puppet agent run, prior run detected' -severity 'INFO'
    if (Is-AggregatorConfiguredCorrectly -aggregator "$aggregator") {{
      Write-Log -message 'correct log aggregator config detected' -severity 'INFO'
    }} else {{
      Set-Aggregator -aggregator "$aggregator"
    }}
  }} else {{
    if (Test-Path $global:puppetLog) {{
      Write-Log -message 'incomplete puppet run detected' -severity 'INFO'
      Remove-Item $global:puppetLog
    }}
    Run-Puppet -hostname $hostname -domain $domain
    Send-Log -logfile $global:puppetLog -subject ('Puppet Agent Report for {{0}}.{{1}}' -f $hostname, $domain)
    Stop-ComputerWithDelay -reason 'userdata golden puppet run complete'
  }}
}} else {{
  if ($hostnameCorrect -ne $true) {{ Set-Hostname -hostname $hostname }}
  if ($domainCorrect -ne $true) {{ Set-Domain -domain $domain }}
  Stop-ComputerWithDelay -reason 'host name changed' -restart
}}

Send-Log -logfile ('{{0}}\Amazon\Ec2ConfigService\Logs\Ec2ConfigLog.txt' -f $env:ProgramFiles) -subject ('EC2 Config Report for {{0}}.{{1}}' -f $hostname, $domain)
Send-Log -logfile $global:runLog -subject ('UserData Run Report for {{0}}.{{1}}' -f $hostname, $domain)
</powershell>
<persist>true</persist>

alternative-xml-edit.ps1

  $sysprepFile = ('{{0}}\Amazon\Ec2ConfigService\sysprep2008.xml' -f $env:ProgramFiles)
  [xml] $xml = Get-Content($sysprepFile)
  $ns = New-Object System.Xml.XmlNamespaceManager($xml.NameTable)
  $ns.AddNamespace("u", 'urn:schemas-microsoft-com:unattend')
  $component = $xml.SelectSingleNode('//u:settings[@pass=specialize]/u:component[@name=Microsoft-Windows-Shell-Setup]', $ns)
  if (-not $component.ComputerName) {{
    $computerNameElement = $xml.CreateElement("ComputerName")
    $computerNameElement.value = "{hostname}"
    $component.AppendChild($computerNameElement)
    Write-Log -message ('computer name inserted to: {{0}}' -f $sysprepFile) -severity 'DEBUG'
  }} else {{
    $component.ComputerName.value = "{hostname}"
    Write-Log -message ('computer name updated in: {{0}}' -f $sysprepFile) -severity 'DEBUG'
  }}

create-ami.sh

#! /bin/bash
 
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# This script will create a windows golden ami.
# If the required DNS entries do not exist, they will be created.
# You must have valid LDAP credentials for invtool in order to create DNS entries.
# You must be connected to the scl3 VPN in order to get a successful run.

# usage:
# try/y-2008:
# $ moztype="y-2008" ./create-ami.sh
# prod/b-2008:
# $ moztype="b-2008" ./create-ami.sh


: ${moztype:?moztype must be set}

sandbox=${sandbox:='/builds/aws_manager'}
aws_source_region=${aws_source_region:='us-east-1'}
aws_target_region=${aws_target_region:='us-west-2'}
target_hostname=${target_hostname:="$moztype-ec2-golden"}

set -e
set -o pipefail
lock_file="$sandbox/$target_hostname.lock"
lockfile -60 -r 3 "$lock_file"
trap "rm -f $lock_file" exit

case "$moztype" in
  "b-2008")
    target_domain="build.releng.use1.mozilla.com"
    target_cname="$target_hostname.build.mozilla.org"
    instance_data="$sandbox/cloud-tools/instance_data/$aws_source_region.instance_data_prod.json"
    ;;
  "y-2008")
    target_domain="try.releng.use1.mozilla.com"
    target_cname="$target_hostname.try.mozilla.org"
    instance_data="$sandbox/cloud-tools/instance_data/$aws_source_region.instance_data_try.json"
    ;;
esac
target_fqdn="$target_hostname.$target_domain"

if [[ $(host $target_cname) = *"not found"* ]]; then
  echo "No DNS entry found for $target_hostname. Creating..."
  target_ip=`$sandbox/bin/python $sandbox/cloud-tools/scripts/free_ips.py -c $sandbox/cloud-tools/configs/$moztype -r $aws_source_region -n1`
  echo "Using IP: $target_ip, FQDN: $target_fqdn, CNAME: $target_cname"
  $sandbox/bin/invtool A create --ip $target_ip --fqdn $target_fqdn --private --description "$target_hostname A record"
  $sandbox/bin/invtool PTR create --ip $target_ip --target $target_fqdn --private --description "$target_hostname reverse mapping"
  $sandbox/bin/invtool CNAME create --fqdn $target_cname --target $target_fqdn --private --description "$target_hostname alias"
  i=1
  while ! host $target_cname; do
    sleep 1m
    let i++
    echo "Slept ${i} minutes, waiting for DNS propagation"
  done
  echo "DNS updated"
fi

cd $sandbox/cloud-tools/scripts
$sandbox/bin/python "$sandbox/cloud-tools/scripts/aws_create_instance.py" -c $sandbox/cloud-tools/configs/$moztype -r $aws_source_region -s aws-releng -k $sandbox/secrets/aws-secrets.json --ssh-key ~/.ssh/aws-ssh-key -i $instance_data --create-ami --ignore-subnet-check --copy-to-region $aws_target_region $target_hostname
echo "Instance created: $target_hostname"

if [ -e "$sandbox/cloud-tools/scripts/$target_hostname.log" ]; then
  if [ -d $sandbox/log ]; then
    python_log="$sandbox/log/$target_hostname.$(date '+%Y%m%d%H%M%S').python.log"
    mv $sandbox/cloud-tools/scripts/$target_hostname.log $python_log
    cat $python_log
  else
    cat $sandbox/cloud-tools/scripts/$target_hostname.log
  fi
fi

create-instance.sh

#! /bin/bash
 
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# This script will create a windows testing instance from the same base ami that is used by golden.
# If the required DNS entries do not exist, they will be created.
# You must have valid LDAP credentials for invtool in order to create DNS entries.
# You must be connected to the scl3 VPN in order to get a successful run.

# usage:
# try/y-2008:
# $ moztype="y-2008" ./create-instance.sh
# prod/b-2008:
# $ moztype="b-2008" ./create-instance.sh


: ${moztype:?moztype must be set}
: ${bug:?bug must be set}

sandbox=${sandbox:='/builds/aws_manager'}
user=${user:=$(whoami)}
email=${sandbox:="[email protected]"}
target_hostname=${target_hostname:="$moztype-ec2-$user"}
target_region=${target_region:="us-east-1"}
secrets=${secrets:="$sandbox/secrets/aws-secrets.json"}

case "$target_region" in
  "us-east-1")
    target_dns_atom="use1"
    ;;
  "us-west-2")
    target_dns_atom="usw2"
    ;;
esac
case "$moztype" in
  "b-2008")
    target_domain="build.releng.$target_dns_atom.mozilla.com"
    target_cname="$target_hostname.build.mozilla.org"
    instance_data="$sandbox/cloud-tools/instance_data/$target_region.instance_data_prod.json"
    ;;
  "y-2008")
    target_domain="try.releng.$target_dns_atom.mozilla.com"
    target_cname="$target_hostname.try.mozilla.org"
    instance_data="$sandbox/cloud-tools/instance_data/$target_region.instance_data_try.json"
    ;;
esac
target_fqdn="$target_hostname.$target_domain"

if [[ $(host $target_cname) = *"not found"* ]]; then
  echo "No DNS entry found for $target_hostname. Creating..."
  target_ip=`$sandbox/bin/python $sandbox/cloud-tools/scripts/free_ips.py -c $sandbox/cloud-tools/configs/$moztype -r $target_region -n1`
  echo "Using IP: $target_ip, FQDN: $target_fqdn, CNAME: $target_cname"
  $sandbox/bin/invtool A create --ip $target_ip --fqdn $target_fqdn --private --description "bug $bug: loaner for $user"
  $sandbox/bin/invtool PTR create --ip $target_ip --target $target_fqdn --private --description "bug $bug: loaner for $user"
  $sandbox/bin/invtool CNAME create --fqdn $target_cname --target $target_fqdn --private --description "bug $bug: loaner for $user"
  i=1
  while ! host $target_cname; do
    sleep 1m
    let i++
    echo "Slept ${i} minutes, waiting for DNS propagation"
  done
  echo "DNS updated"
fi

cd $sandbox/cloud-tools/scripts
python "$sandbox/cloud-tools/scripts/aws_create_instance.py" \
  -c "$sandbox/cloud-tools/configs/$moztype" \
  -r $target_region \
  -s aws-releng \
  --loaned-to $email --bug $bug -k $secrets \
  --ssh-key ~/.ssh/aws-ssh-key \
  -i $instance_data $target_hostname

if [ -e "$sandbox/cloud-tools/scripts/$target_hostname.log" ]; then
  if [ -d $sandbox/log ]; then
    python_log="$sandbox/log/$target_hostname.$(date '+%Y%m%d%H%M%S').python.log"
    mv $sandbox/cloud-tools/scripts/$target_hostname.log $python_log
    cat $python_log
  else
    cat $sandbox/cloud-tools/scripts/$target_hostname.log
  fi
fi