blueGreenDeploy.ps1

blueGreenDeploy.ps1

<#
.SYNOPSIS
    Command line utility for initiating blue-green deployments, to be called by a build step, or by another script, in a Jenkins job.
   
.PARAMETER  region
	Optional/has default value.  Specifies the AWS region, e.g. 'us-east-1'.  See script Param section for allowed values.												

.PARAMETER  environment
    Required.  Specifies a friendly name for the environment, e.g. 'dev'.  See script Param section for allowed values.

.PARAMETER  product
    Required.  Specifies a friendly name for a product, e.g. 'fms'.  See script Param section for allowed values.

.PARAMETER  uid
    Required.  Specifies the uid for the product stack, e.g. 'uaa'.

.PARAMETER  zip_name
    Required.  Specifies the value to apply to the 'deploymentZip' tag, to bind the new ASG to a single app version, e.g. 'dev-lms-49-2015-10-28.zip'.

.PARAMETER  accessKey
    Optional.  Specifies an AWS access key to use for running locally/testing.

.PARAMETER  secretKey
    Optional.  Specifies an AWS secret key to use for running locally/testing.

.EXAMPLE
	.\deploy.ps1 -environment dev -product fms -uid uaa -zip_name dev-lms-49-2015-10-28.zip

.NOTES
	DEPENDS: AWS Tools for Windows PowerShell: 3.1.23.0 or greater

.TODO
    - add granular $rollBack levels and modify RollBack function to handle them
    - make detachment from live elb conditional in RollBack
    - make check for new scaling events occur during each CheckInService iteration, so new values can be applied as early as possible
#>


Param
(
    [Parameter(Mandatory=$False)]
    [ValidateSet("us-east-1","us-west-1","us-west-2","ap-northeast-1","ap-southeast-1","ap-southeast-2","eu-central-1","eu-west-1","sa-east-1",IgnoreCase = $false)]
    [string]$region = "us-east-1",
    [Parameter(Mandatory=$True)]
    [ValidateSet("dev","test","testfull","preview","prod",IgnoreCase = $false)]
    [string]$environment,
    [Parameter(Mandatory=$True)]
    [ValidateSet("fms","lms","cms","none",IgnoreCase = $false)]
    [string]$product,
    [Parameter(Mandatory=$True)]
    [string]$uid,
    [Parameter(Mandatory=$True)]
    [string]$zip_name,
    [Parameter(Mandatory=$False)]
    [string]$accessKey,
    [Parameter(Mandatory=$False)]
    [string]$secretKey
)


# pretty logging for Jenkins console output.
function Log ($indentLevel, $logLevel, $logMessage)
{
    try
    {
        $now = get-date -Format "hh:mm:ss"
        $indent = "`t" * $indentLevel
        Write-Host $now $loglevel$indent$logMessage  
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "Log" $exMsg
    }
}


# to ensure API calls that don't return anything actually got made
function awsLog ($indentLevel)
{
    try
    {
        $logLevel = "INFO"
        $requestId = $AWSHistory.LastServiceResponse.ResponseMetadata.RequestId
        $requestStatusCode = $AWSHistory.LastServiceResponse.HttpStatusCode
        if ($requestStatusCode -ne "OK")
        {
            $logLevel = "WARN"
        }
        Log $indentLevel $logLevel "AWS response: $requestStatusCode / Request ID: $requestId"
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "awsLog" $exMsg
    }
}


function HandleException($function,$exceptionMessage)
{
    $msg = "Exception in $($function): $exceptionMessage, exiting."
    Log 1 FATAL $msg
    throw $msg
    exit 1
}


function Setup ($region)
{
    try
    {
        # remove any lingering AWS sessions, to ensure that Jenkins role or passed key params are used
        Clear-AWSDefaults
        Clear-AWSCredentials
        Set-DefaultAWSRegion -Region $region
        Log 2 INFO "script environment ready"
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "Setup" $exMsg
    }
}


function CheckStackState
{
    try
    {
        $isBad = $false
        $asgs = Get-ASAutoScalingGroup | where {$_.AutoScalingGroupName.StartsWith("$environment-$product")}
        $liveAsgCount = ($asgs | where {$_.DesiredCapacity -ne 0}).count
        if ($liveAsgCount -gt 1)
        {
            Log 2 FATAL "there are $liveAsgCount ASGs with nonzero desired capacity for product $product in environment $environment"
            $isBad = $true
        }
        $activeAsgCount = ($asgs | where {$_.Tags.value.Contains("active")}).count
        if ($activeAsgCount -gt 1)
        {
            Log 2 FATAL "there are $activeAsgCount ASGs with state = 'active' for product $product in environment $environment"
            $isBad = $true
        }
        $tempElb = Get-ELBLoadBalancer | where {$_.LoadBalancerName -eq "$environment-$product-elb-temp"}
        if ($tempElb)
        {
            Log 2 FATAL "at least one ELB with name $environment-$product-elb-temp already exists"
            $isBad = $true
        }
        $tempElbStack = Get-CFNStack | where {$_.StackName -eq "$environment-$product-temp-elb"}
        if ($tempElbStack)
        {
            # very rarely, DeleteTempElb at the end of the script results in DELETE_FAILED
            # in those cases, we'll just delete it again.
            # incuding "DELETE_IN_PROGRESS" here so back-to-back deployments function correctly as well
            if (($tempElbStack.StackStatus.Value -eq "DELETE_FAILED") -or ($tempElbStack.StackStatus.Value -eq "DELETE_IN_PROGRESS"))
            {
                Log 2 WARN "CFN stack $environment-$product-temp-elb is in state $($tempElbStack.StackStatus.Value) from previous deployment, cleaning this up"
                $isBad = DeleteTempElb "recheck"  
            }
            else
            {
                Log 2 FATAL "CFN stack $environment-$product-temp-elb is in an unexpected state"
                $isBad = $true
            }        
        }

        # check that current live ASG instances are all InService
        $instanceStates = Get-ELBInstanceHealth -LoadBalancerName "$environment-$product-elb-$uid" | select State
        foreach ($instanceState in $instanceStates)
        {
            if ($instanceState.State -ne "InService")
            {
                Log 2 FATAL "elb for active ASG $environment-$product-elb-$uid has one or more instances not InService."
                $isBad = $true
            }
        }


        if ($isBad)
        {
            Log 2 FATAL "stack is not in acceptable state for deployment"
            Log 2 FATAL "is a previous deployment still in progress / teardown failed?"
            HandleException "CheckStackState" "FAILED"
        }

        Log 2 INFO "stack ready for deployment"
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "CheckStackState" $exMsg
    }
}


function GetAsg ($status)
{
    try
    {
        sleep -Seconds 3 # buffer to allow previous updates to asg values to propagate in AWS before making back-to-back calls
        $asgs = Get-ASAutoScalingGroup | where {$_.AutoScalingGroupName.StartsWith("$environment-$product")} #must refresh every time called
        foreach ($asg in $asgs)
        {
            $tags = $asg.Tags
            $asgStatus = ($tags | where {$_.Key -eq "status"}).Value
            if ($asgStatus -eq $status  )
            {
                return $asg
            } 
        }
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "GetAsg" $exMsg
    }
}


function UpdateInactiveASG ($activeAsg,$inactiveAsg)
{
    try
    {
        $inactiveAsgName = $inactiveAsg.AutoScalingGroupName
        $wasUpdated = $false
        if (($inactiveAsg.MaxSize) -ne ($activeAsg.MaxSize))
        {
            Log 2 INFO "updating inactive ASG MaxSize: $($inactiveAsg.MaxSize) to match active ASG MaxSize: $($activeAsg.MaxSize)"
            Update-ASAutoScalingGroup -AutoScalingGroupName $inactiveAsgName -MaxSize $activeAsg.MaxSize
            awsLog 3
            $wasUpdated = $true
        }
        if (($inactiveAsg.MinSize) -ne ($activeAsg.MinSize))
        {
            Log 2 INFO "updating inactive ASG MinSize: $($inactiveAsg.MinSize) to match active ASG MinSize: $($activeAsg.MinSize)"
            Update-ASAutoScalingGroup -AutoScalingGroupName $inactiveAsgName -MinSize $activeAsg.MinSize
            awsLog 3
            $wasUpdated = $true
        }
        if (($inactiveAsg.DesiredCapacity) -ne ($activeAsg.DesiredCapacity))
        {
            Log 2 INFO "updating inactive ASG DesiredCapacity: $($inactiveAsg.DesiredCapacity) to match active ASG DesiredCapacity: $($activeAsg.DesiredCapacity)"
            Update-ASAutoScalingGroup -AutoScalingGroupName $inactiveAsgName -DesiredCapacity $activeAsg.DesiredCapacity
            awsLog 3
            $wasUpdated = $true
        }

        return $wasUpdated
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "UpdateInactiveASG" $exMsg
    }
}

function UpdateAsgTag ($asg,$tagKey,$tagValue)
{
    try
    {
        $asgTags = $asg.Tags
        $asgName = $asg.AutoScalingGroupName
        $currentTagValue = ($asgTags | where {$_.Key -eq $tagKey}).Value
        if ($currentTagValue -ne $tagValue)
        {
            # ASG names and zips are long values, broken up to avoid line wrapping
            Log 2 INFO "updating tag for ASG: $asgName"
            Log 3 INFO "tag: $tagKey, currentValue: $currentTagValue, newValue: $tagValue"
            Set-ASTag -Tag @( @{ResourceType="auto-scaling-group"; ResourceId=$asgName; Key=$tagKey; Value=$tagValue; PropagateAtLaunch=$true} )
        }
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "UpdateAsgTag" $exMsg
    }
}


function CreateTempElb
{
    try
    {
        $elbStack = Get-CFNStack | where {$_.StackName.StartsWith("$environment-$product-$uid-elb")}
        $tempElbStackName = "$environment-$product-temp-elb"
        $stackParams = $elbStack.Parameters
        $uidParam = $stackParams | where {$_.ParameterKey -eq "uidParameter"}
        $uidParam.ParameterValue = "temp"
        $cfnStatusCheckInterval = 15 #seconds
        $templateUrl = "https://s3.amazonaws.com/$environment-cfntemplates-$uid/$product/elb.json"

        $resp = New-CFNStack -StackName $tempElbStackName -Parameter $stackParams -TemplateURL $templateUrl -Capabilities "CAPABILITY_IAM"
        Log 2 INFO "CFN response: $resp"

        do
        {
            sleep -Seconds $cfnStatusCheckInterval
            $stack = Get-CFNStack -StackName $tempElbStackName
            $stackStatus = $stack.StackStatus
            Log 2 INFO "...$stackStatus"
        }
        until
        (
            $stackStatus -eq "CREATE_COMPLETE"
        )
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "CreateTempElb" $exMsg
    }
}


function UpdateElb ($asg,$elbName,$action)
{
    try
    {
        if ($action -eq "mount")
        {
            Add-ASLoadBalancer -LoadBalancerName $elbName -AutoScalingGroupName $asg.AutoScalingGroupName
        }
        if ($action -eq "dismount")
        {
            Dismount-ASLoadBalancer -LoadBalancerName $elbName -AutoScalingGroupName $asg.AutoScalingGroupName
        }
        Log 2 INFO "action: $action"
        Log 2 INFO "ELB: $elbName"
        Log 2 INFO "ASG: $($asg.AutoScalingGroupName)"
        awsLog 3
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "UpdateElb" $exMsg
    }
}


function CheckInService ($elbName,$expectedInstances)
{
    try
    {
        $retries = 1
        $elbInstanceHealthCheckInterval = 15 #seconds
        $elbInstanceHealthRetries = 100
        do
        {
            sleep -Seconds $elbInstanceHealthCheckInterval
            $instances = Get-ELBInstanceHealth -LoadBalancerName $elbName
            $instancesInService = $instances | where {$_.State -eq "InService"}
            $retryString = $retries.ToString("000")
            Log 2 INFO "retry $($retryString)/$($elbInstanceHealthRetries):`t$($instancesInService.count)/$expectedInstances instances in service"
            $retries += 1
        }
        until
        (
            (($instancesInService.count -eq $expectedInstances) -or ($retries -ge $elbInstanceHealthRetries))
        )

        if ($retries -ge $elbInstanceHealthRetries)
        {
            RollBack
        }
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "CheckInService" $exMsg
    }
}


function UpdateAutoScaling ($asg,$action)
{
    try
    {
        Log 2 INFO "action: $action"
        Log 2 INFO "ASG: $($asg.AutoScalingGroupName)"
        if ($action -eq "suspend")
        {
            Suspend-ASProcess -AutoScalingGroupName $asg.AutoScalingGroupName
        }
        if ($action -eq "resume")
        {
            Resume-ASProcess -AutoScalingGroupName $asg.AutoScalingGroupName
        }
        if ($action -eq "noCloudWatchAlarms")
        {
            Suspend-ASProcess -AutoScalingGroupName $asg.AutoScalingGroupName -ScalingProcess AlarmNotification
        }
        awsLog 3
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "UpdateAutoScaling" $exMsg
    }
}


function PurgeAsg ($asg)
{
    try
    {
        $asgName = $asg.AutoScalingGroupName
        Log 2 INFO "updating inactive ASG MinSize: $($asg.MinSize) to 0"
        Update-ASAutoScalingGroup -AutoScalingGroupName $asgName -MinSize 0
        awsLog 3

        Log 2 INFO "updating inactive ASG DesiredCapacity: $($asg.DesiredCapacity) to 0"
        Update-ASAutoScalingGroup -AutoScalingGroupName $asgName -DesiredCapacity 0
        awsLog 3

        Log 2 INFO "updating inactive ASG MaxSize: $($asg.MaxSize) to 0"
        Update-ASAutoScalingGroup -AutoScalingGroupName $asgName -MaxSize 0
        awsLog 3
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "PurgeAsg" $exMsg
    }
}


function DeleteTempElb ($recheck)
{
    try
    {
        $tempElbStackName = "$environment-$product-temp-elb"
        Log 2 INFO "deleting CFN stack $tempElbStackName"
        Remove-CFNStack -StackName $tempElbStackName -Force
        awsLog 3

        if ($recheck -eq "recheck")
        {
            $retryThreshold = 20 #10m @ 30s.  Usually takes 2m - 5m
            do
            {
                $tempElbStack = Get-CFNStack | where {$_.StackName.Contains($tempElbStackName)}
                if($tempElbStack)
                {
                    Log 3 INFO "...$($tempElbStack.StackStatus.Value)"
                    if ($($tempElbStack.StackStatus.Value) -eq "DELETE_FAILED")
                    {
                        Log 3 INFO "...(we'll see about that)"
                        Remove-CFNStack -StackName $tempElbStackName -Force
                    }
                }
                sleep -Seconds 30
                $retries += 1
            }
            until
            (
                ($tempElbStack -eq $null) -or ($retries -ge $retryThreshold)
            )

            if ($tempElbStack -eq $null)
            {
                Log 3 INFO "...DELETED"
                return $false #sets $isBad = $false in CheckStackState because tempElbStack is now gone.
            }
            if ($retries -ge $retryThreshold)
            {
                return $true #sets $isBad = $true in CheckStackState because it refused to die.  This should never happen.
            }
        }
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "DeleteTempElb" $exMsg
    }
}


function RollBack
{
    try
    {
        Log 1 WARN "One or more instances unhealthy after retry threshold was reached.  Rolling back."
        
        $inactiveAsg = GetAsg inactive
        $activeAsg = GetAsg active

        UpdateAutoScaling $inactiveAsg resume
        UpdateAutoScaling $activeAsg resume
        PurgeAsg ($inactiveAsg)

        $tempElbName = "$environment-$product-elb-temp"
        UpdateElb $inactiveAsg $tempElbName dismount

        #this throws if not actually attached, make it conditional (if attached)
        #$activeElbName = "$environment-$product-elb-$uid"
        #UpdateElb $inactiveAsg $activeElbName dismount

        DeleteTempElb "noRecheck"

        Log 1 INFO "Rollback complete, script exiting"
        # "ROLLBACK" | Out-File -Append -FilePath .\deployTestLog.csv # FOR TESTING
        exit 1
    }
    Catch [Exception]
    {
        $exMsg = $_.Exception.Message
        HandleException "RollBack" $exMsg
    }
}


# --------------------------------- script entry point ---------------------------------
# setup
Log 1 INFO "Setup:"
Setup $region


# if keys are passed, use them.
Set-DefaultAWSRegion -Region $region
if (($accessKey) -and ($secretKey))
{
    Initialize-AWSDefaults -AccessKey $accessKey -SecretKey $secretKey -Region $region
}


# ensure stack is in good state for a deployment
Log 1 INFO "CheckStackState:"
CheckStackState


# get asgs
Log 1 INFO "GetAsgs:"
$activeAsg = GetAsg active
Log 2 INFO "active ASG: $($activeAsg.AutoScalingGroupName)"
$inactiveAsg = GetAsg inactive
Log 2 INFO "inactive ASG: $($inactiveAsg.AutoScalingGroupName)"


# clone active elb into a temp elb and attach to inactive asg to satisfy health check
# this is done like this because each ELB is ~$20/month
Log 1 INFO "CreateTempElb:"
CreateTempElb


# mount temp elb on inactive asg
$tempElbName = "$environment-$product-elb-temp"
Log 1 INFO "UpdateElb:"
UpdateElb $inactiveAsg $tempElbName mount


# disable autoscaling triggered by low CPU/mem/network CW alarms on inactive asg 
# to prevent unwanted scale-down events while asg is waiting for instances to come in service
Log 1 INFO "UpdateAutoScaling:"
UpdateAutoScaling $inactiveAsg noCloudWatchAlarms


# update inactive asg with new deployment.zip tag and same des/min/max as active asg
Log 1 INFO "UpdateInactiveASG:"
UpdateAsgTag $inactiveAsg deploymentZip "$product/$zip_name"
$wasUpdated = UpdateInactiveASG $activeAsg $inactiveAsg


# ensure all instances behind temp elb are inService
# if activeAsg scales during this check, adjust and recheck before continuing
Log 1 INFO "CheckInService on ELB $($tempElbName):"
do
{
    $inactiveAsg = GetAsg inactive # to refresh DesiredCapacity
    $expectedInstances = $inactiveAsg.DesiredCapacity
    CheckInService $tempElbName $expectedInstances
    $activeAsg = GetAsg active # to refresh desired/min/max values in case there was a scaling event during previous CheckInService
    Log 1 INFO "Checking for min/max/desired changes caused by new scaling events in active ASG"
    $wasUpdated = UpdateInactiveASG $activeAsg $inactiveAsg
    if (!$wasUpdated)
    {
        Log 1 INFO "No new scaling events found, continuing with deployment"
    }  
}
until
(
    $wasUpdated -eq $False
)


# suspend scaling on active asg, not done earlier because time window required by CheckInService is too long 
# to disable autoscaling.  Done here to ensure scaling events from this point forward do not interfere w deployment.
# since elb heath checks are identical, this window is always be very brief ( ~30 seconds in testing)
Log 1 INFO "UpdateAutoScaling:"
UpdateAutoScaling $activeAsg suspend


# mount active elb on inactiveAsg
$activeElbName = "$environment-$product-elb-$uid"
Log 1 INFO "UpdateElb:"
UpdateElb $inactiveAsg $activeElbName mount


# ensure all instances behind active elb are inService
Log 1 INFO "CheckInService on ELB $($activeElbName):"
$expectedInstances = $inactiveAsg.DesiredCapacity + $activeAsg.DesiredCapacity
CheckInService $activeElbName $expectedInstances


# update asg status tags, and refresh asg objects
Log 1 INFO "UpdateAsgTag:"
UpdateAsgTag $inactiveAsg status active
UpdateAsgTag $activeAsg status inactive
$activeAsg = GetAsg active
$inactiveAsg = GetAsg inactive


# turn autoscaling back on for activeAsg
# turn autoscaling back on for inactiveAsg so its in a consistent state for next deployment
Log 1 INFO "UpdateAutoScaling:"
UpdateAutoScaling $activeAsg resume
UpdateAutoScaling $inactiveAsg resume


# dismount active elb from the inactiveAsg, and temp elb from the activeAsg
Log 1 INFO "UpdateElb:"
UpdateElb $inactiveAsg $activeElbName dismount
UpdateElb $activeAsg $tempElbName dismount


# set inactiveAsg des/min/max to 0
Log 1 INFO "PurgeAsg:"
PurgeAsg $inactiveAsg


# delete temp elb stack
# noRecheck keeps it from blocking in Jenkins
Log 1 INFO "DeleteTempElb:"
DeleteTempElb "noRecheck"


# have a nice day!
Log 1 INFO "Deployment complete, script exiting"
#"SUCCESS" | Out-File -Append -FilePath .\deployTestLog.csv # FOR TESTING
exit 0