Created
September 15, 2015 19:55
-
-
Save ezeeetm/9041be22501b524d1967 to your computer and use it in GitHub Desktop.
write your own
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<# | |
.SYNOPSIS | |
Command line utility for scheduled DR backups to an isolated offsite AWS account. | |
Logs to .\dailySnapShotLogs | |
.EXAMPLE | |
.\dailySnapShots.ps1 | |
Starts the long running script | |
.NOTES | |
AUTHOR: Eric Miller | |
VERSION: 1.0 | |
DATE: 26MAY2015 | |
DEPENDS: .\logging.ps1 | |
#> | |
# init | |
clear | |
. .\logging.ps1 | |
$now = Get-Date | |
write-host "$now dailySnapShots..." | |
# config | |
$triggerHour = 23 # scheduled hour, server time (UTC) in 24h format | |
$createVolumePermission = New-Object Amazon.EC2.Model.CreateVolumePermission # custom AWS object used for cross-account snapshot permissions | |
$createVolumePermission.UserId = 'REDACTED' # DR account ID | |
$authToken = "REDACTED" # hipchat | |
$roomId = "REDACTED" | |
$from = 'DR Agent' | |
$prodCreds = @('PROD','REDACTED','REDACTED') | |
$drCreds = @('DR','REDACTED','REDACTED') | |
$sourceRegion = 'us-east-1' # use different source/dest region for DR | |
$destRegion = 'us-west-1' | |
$drRetention = 30 # period to retain snapshot copies in DR account, in days | |
$snapShotCheckWait = 60 # delay between snapshot completion checks, in seconds (prevents excessive 'pending' log entries) | |
$copyCheckWait = 60 # delay between copy completion checks, in seconds (prevents excessive 'pending' log entries) | |
# /config | |
# hashtable of mission critical volumes in format "Description" = "vol-########" | |
$volumes = @{ | |
"REDACTED" = "vol-REDACTED"; | |
"REDACTED" = "vol-REDACTED"} | |
function handleException($function,$exceptionMessage) | |
{ | |
$message = "Exception in $($function): $exceptionMessage" | |
log WARNING $message | |
hipChat $message 'red' | |
$errorCount += 1 | |
} | |
function completeSnapshots ($snapshots,$checkWait,$region) | |
{ | |
try | |
{ | |
foreach ($snapshot in $snapshots) | |
{ | |
$state = "" | |
do | |
{ | |
$state = $(Get-EC2Snapshot -SnapshotId $snapshot -Region $region).State | |
log INFO "$snapshot $state" | |
sleep -Seconds $checkWait | |
} | |
until($state -eq "completed") | |
} | |
} | |
Catch [Exception] | |
{ | |
$exMsg = $_.Exception.Message | |
handleException "completeSnapshots" $exMsg | |
} | |
} | |
function createSourceSnapshots ($volumes) | |
{ | |
try | |
{ | |
# create snapshot in source account | |
$snapshots =@() # to ensure any failure below does not result in creating new copies of source snapshots from a previous invocation, and to cast as an array (otherwise += below treats as string) | |
foreach ($key in $volumes.Keys) | |
{ | |
$timeStamp = Get-Date -Format MMddHHmm | |
$desc = "$key DR/daily $timeStamp" | |
$volId = $volumes.$key | |
$resp = New-EC2Snapshot -VolumeId $volId -Description $desc | |
log INFO "$($resp.Description), $($resp.SnapshotId), $($resp.StartTime), $($resp.State)" | |
$snapshots += $resp.SnapshotId | |
} | |
# ensure snapshots are completed | |
completeSnapshots $snapshots $snapShotCheckWait $sourceRegion | |
# apply create volume permission for destination account | |
foreach ($snapshot in $snapshots) | |
{ | |
Edit-EC2SnapshotAttribute -SnapshotId $snapshot -CreateVolumePermission_Add $createVolumePermission | |
} | |
return $snapshots | |
} | |
Catch [Exception] | |
{ | |
$exMsg = $_.Exception.Message | |
handleException "createSourceSnapshots" $exMsg | |
} | |
} | |
function copySnapshots ($snapshots) | |
{ | |
try | |
{ | |
$concurrentCopies = @() | |
foreach ($snapshot in $snapshots) | |
{ | |
# get original snapshot metadata and copy each snapshot to DR region, 5 at a time (this is the concurrent snapshot copy limit and cannot be increased) | |
$snapshot = Get-EC2Snapshot -SnapshotId $snapshot | |
$desc = $snapshot.Description | |
$snapshotId = $snapshot.SnapshotId | |
$copySnapshotId = Copy-EC2Snapshot -SourceSnapshotId $snapshotId -SourceRegion $sourceRegion -Description $desc -DestinationRegion $destRegion | |
$concurrentCopies += $copySnapshotId | |
$destSnapshots += $copySnapshotId | |
if ($concurrentCopies.count -eq 5) | |
{ | |
completeSnapshots $concurrentCopies $copyCheckWait $destRegion | |
$concurrentCopies = @() | |
} | |
} | |
completeSnapshots $concurrentCopies $copyCheckWait $destRegion # to catch the last few snapshots if batch < 5 | |
} | |
Catch [Exception] | |
{ | |
$exMsg = $_.Exception.Message | |
handleException "copySnapshots" $exMsg | |
} | |
} | |
function enforceDRRetention | |
{ | |
try | |
{ | |
log INFO "Checking snapshot copy retention..." | |
$snaps = Get-EC2Snapshot -Owner self -Region us-west-1 | |
$expiredSnaps = $null | |
$expiredSnaps = $snaps | where {$_.startTime -le $(get-date).AddDays(-$drRetention)} | |
foreach ($expiredSnap in $expiredSnaps) | |
{ | |
$snapshotId = $expiredSnap.SnapshotId | |
log INFO "Deleting $snapshotId by retention policy" | |
Remove-EC2Snapshot -SnapshotId $snapshotId -Force | |
} | |
if ($expiredSnaps -eq $null) | |
{ | |
log INFO "No expired snapshots found, continuing." | |
} | |
} | |
Catch [Exception] | |
{ | |
$exMsg = $_.Exception.Message | |
handleException "enforceDRRetention" $exMsg | |
} | |
} | |
function deleteSourceSnaps ($snapshots) | |
{ | |
try | |
{ | |
foreach ($snapshot in $snapshots) | |
{ | |
log INFO "Deleting source $snapshot" | |
Remove-EC2Snapshot -SnapshotId $snapshot -Force | |
} | |
} | |
Catch [Exception] | |
{ | |
$exMsg = $_.Exception.Message | |
handleException "deleteSourceSnaps" $exMsg | |
} | |
} | |
function hipChat ($message, $color) | |
{ | |
try | |
{ | |
$url = "https://api.hipchat.com/v1/rooms/message?auth_token=$authToken&room_id=$roomId&from=$from&message=$message¬ify=1&message_format=text&color=$color" | |
Invoke-WebRequest -Uri $url -Method Post | out-null | |
} | |
Catch [Exception] | |
{ | |
$exMsg = $_.Exception.Message | |
handleException "hipChat" $exMsg | |
} | |
} | |
# script entry point #################################################### | |
while($true) | |
{ | |
try | |
{ | |
$time = get-date | |
$errorCount = 0 | |
if ($time.Hour -eq $triggerHour) | |
{ | |
$message = 'DR Copy initiated by schedule' | |
log INFO $message | |
hipChat $message 'gray' | |
# use PROD account, create source snapshots. Doing the session in a function was a PITA, so put it here. | |
$creds = $prodCreds; $accountDesc = $creds[0]; $AK = $creds[1]; $SK = $creds[2] | |
log INFO "initializing AWS session with $accountDesc account" | |
Initialize-AWSDefaults -AccessKey $AK -SecretKey $SK -Region us-east-1 | |
log INFO "initializing AWS session with $accountDesc account complete" | |
$snapshots = createSourceSnapshots $volumes | |
# use DR account, create copies of snapshots in destination account/region | |
$creds = $drCreds; $accountDesc = $creds[0]; $AK = $creds[1]; $SK = $creds[2] | |
log INFO "initializing AWS session with $accountDesc account" | |
Initialize-AWSDefaults -AccessKey $AK -SecretKey $SK -Region us-east-1 | |
log INFO "initializing AWS session with $accountDesc account complete" | |
copySnapshots $snapshots | |
# while still in DR account, apply snapshot copy retention policy | |
enforceDRRetention | |
# switch back to prod account and delete source snapshots to avoid double storage costs | |
$creds = $prodCreds; $accountDesc = $creds[0]; $AK = $creds[1]; $SK = $creds[2] | |
log INFO "initializing AWS session with $accountDesc account" | |
Initialize-AWSDefaults -AccessKey $AK -SecretKey $SK -Region us-east-1 | |
log INFO "initializing AWS session with $accountDesc account complete" | |
deleteSourceSnaps $snapshots | |
$message = "DR Copy completed with $errorCount error(s)" | |
log INFO $message | |
hipChat $message 'gray' | |
sleep -Seconds 7200 # two hours, to ensure only one run per 24h | |
} | |
} | |
Catch [Exception] | |
{ | |
$exMsg = $_.Exception.Message | |
handleException "mainLoop" $exMsg | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment