Manage AWS EBS's backup script

config.yml

# Target Region
:region: ap-northeast-1

# Target Accounts
:accounts:
  test_account:
    :access_key_id: ACCESS_KEY_ID
    :secret_access_key: SECRET_ACCESS_KEY

libs.rb

##
# define functions
##
def elect_snapshot_targets(instances, volumes)
  target_volumes = Array.new
  instances.each do |instance|
    next if instance[:tags]["NoEbsAutoBackup"] == "True"
    next unless instance[:block_device_mappings]

    instance[:block_device_mappings].each do |attached_volume|
      target_volume = volumes.find do |volume|
        volume[:aws_id] == attached_volume[:ebs_volume_id]
      end

      next if target_volume[:tags]["NoEbsAutoBackup"] == "True"
      target_volume[:aws_instance_name] = nil
      target_volume[:aws_instance_name] = instance[:tags]["Name"] if instance[:tags]["Name"]
      target_volumes << target_volume
    end
  end
  return target_volumes
end

def grouping_snapshots_by_volume_id(snapshots)
  grouped_snapshots = Hash.new
  snapshots.each do |snapshot|
    grouped_snapshots[snapshot[:aws_volume_id]] = Array.new unless grouped_snapshots[snapshot[:aws_volume_id]]
    grouped_snapshots[snapshot[:aws_volume_id]] << snapshot
  end
  return grouped_snapshots
end

def detect_aged_snapshots(grouped_snapshots, volumes)
  all_aged_snapshot_ids = Array.new
  grouped_snapshots.each_pair do |volume_id, snapshots|
    target_volume = volumes.find do |volume|
      volume[:aws_id] == volume_id
    end

    next unless target_volume

    retention = $default_retention
    retention = target_volume[:tags]["AutoBackupRetention"].to_i if target_volume[:tags]["AutoBackupRetention"].to_i > 0

    snapshots = snapshots.sort_by{|val| val[:aws_started_at]}.reverse
    aged_snapshots = snapshots[retention, snapshots.length]

    next unless aged_snapshots

    aged_snapshots.each do |aged_snapshot|
      all_aged_snapshot_ids << aged_snapshot[:aws_id]
    end
  end
  return all_aged_snapshot_ids
end

def detect_deleted_volumes(grouped_snapshots, volumes)
  deleted_volume_ids = Array.new
  grouped_snapshots.each_pair do |volume_id, snapshots|
    target_volume = volumes.find do |volume|
      volume[:aws_id] == volume_id
    end

    next if target_volume
    deleted_volume_ids << volume_id
  end
  return deleted_volume_ids
end

def detect_unused_volumes(volumes)
    unused_volumes = volumes.find_all do |volume|
      volume[:aws_status] == "available"
    end

    unused_volume_ids = Array.new
    unused_volumes.each do |unused_volume|
      unused_volume_ids << unused_volume[:aws_id]
    end

    return unused_volume_ids
end

manage_backup.rb

#!/usr/bin/ruby

require 'rubygems'
require 'right_aws'
require 'yaml'
require 'pp'
require "time"

require "libs.rb"

$default_retention = 7

##
# main proccess
##
config = YAML.load(File.open('config.yml'))

config[:accounts].each_pair do |account, params|
  begin
    puts "[Info] <#{account}> Start Backup"

    ec2 = RightAws::Ec2.new(
      params[:access_key_id], 
      params[:secret_access_key], 
      :region => config[:region],
      :logger => Logger.new('/dev/null')
    )

    instances = ec2.describe_instances(:filters => {'instance-state-name' => 'running'})
    volumes   = ec2.describe_volumes

    # Create Snapshot.
    target_volumes = elect_snapshot_targets(instances, volumes)
    target_volumes.each do |volume|
      snapshot = ec2.create_snapshot(volume[:aws_id], "<instance_name:#{volume[:aws_instance_name]}> <volume_id:#{volume[:aws_id]}>")
      tags     = ec2.create_tags(snapshot[:aws_id], {"ManagedByAutoBackup" => "True"})
      puts "[Info] <#{account}> Created snapshot <instance_name:#{volume[:aws_instance_name]}> <volume_id:#{volume[:aws_id]}> <snapshot_id:#{snapshot[:aws_id]}>"
    end

    snapshots = ec2.describe_snapshots(:owner => ['self'], :filters => {'tag:ManagedByAutoBackup' => 'True'})
    grouped_snapshots = grouping_snapshots_by_volume_id(snapshots)

    # Delete aged Snapshot.
    aged_snapshot_ids = detect_aged_snapshots(grouped_snapshots, volumes)
    aged_snapshot_ids.each do |aged_snapshot_id|
      ec2.delete_snapshot(aged_snapshot_id)
      puts "[Info] <#{account}> Deleted aged snapshot <snapshot_id:#{aged_snapshot_id}>"  
    end

    # Detect deleted volume.
    deleted_volume_ids = detect_deleted_volumes(grouped_snapshots, volumes)
    deleted_volume_ids.each do |deleted_volume_id|
      warn "[info] <#{account}> Detected deleted volume <volume_id:#{deleted_volume_ids}>"
    end

    # Detect unused volume.
    unused_volume_ids = detect_unused_volumes(volumes)
    unused_volume_ids.each do |unused_volume_id|
      warn "[info] <#{account}> Detected unused volume <volume_id:#{unused_volume_id}>"
    end

  rescue => ex
    warn "[Error] #{account} Error has Occured"
    warn "[Error] #{account} #{ex.message}"
    next
  end
end

test_libs.rb

require 'test/unit'
require 'pp'

require "libs.rb"

$default_retention = 3

class Test_libs < Test::Unit::TestCase
  def test_elect_snapshot_target
    # Test for EBS-backed
    instances = [
      {:aws_instance_id=>"i-aaa", :block_device_mappings=> [{:ebs_volume_id=>"vol-aaa-01"}], :tags=> {"Name"=>"inst-aaa"                        }}, 
      {:aws_instance_id=>"i-bbb", :block_device_mappings=> [{:ebs_volume_id=>"vol-bbb-01"}], :tags=> {"Name"=>"inst-bbb", "NoEbsAutoBackup"=>"True"}}
    ]
    volumes = [
      {:aws_id=>"vol-aaa-01", :tags=>{}                          }, 
      {:aws_id=>"vol-aaa-02", :tags=>{"NoEbsAutoBackup"=>"True"}    }, 
      {:aws_id=>"vol-bbb-01", :tags=>{}                          }, 
      {:aws_id=>"vol-bbb-02", :tags=>{"NoEbsAutoBackup"=>"True"}    },
      {:aws_id=>"vol-bbb-03", :tags=>{"AutoBackupRetention"=>"3"}}
    ]
    expect = [
      {:aws_id=>"vol-aaa-01", :tags=>{}, :aws_instance_name=>"inst-aaa"}
    ]
    assert_equal(expect, elect_snapshot_targets(instances, volumes))

    # Test for instance store
    instances = [
      {:aws_instance_id=>"i-aaa", :tags=> {"Name"=>"inst-aaa"}}, 
    ]
    volumes = [
      {:aws_id=>"vol-aaa-01", :tags=>{}}, 
    ]
    expect = []
    assert_equal(expect, elect_snapshot_targets(instances, volumes))
  end

  def test_grouping_snapshots_by_volume_id
    snapshots = [ 
      {:aws_id=>"snap-aaa-01-01", :aws_volume_id=>"vol-aaa-01"}, 
      {:aws_id=>"snap-aaa-02-01", :aws_volume_id=>"vol-aaa-02"}, 
      {:aws_id=>"snap-bbb-01-01", :aws_volume_id=>"vol-bbb-01"},
      {:aws_id=>"snap-aaa-01-02", :aws_volume_id=>"vol-aaa-01"} 
    ]

    expect = {
      "vol-bbb-01"=> [
        {:aws_volume_id=>"vol-bbb-01", :aws_id=>"snap-bbb-01-01"} 
      ],                              
      "vol-aaa-01"=> [                
        {:aws_volume_id=>"vol-aaa-01", :aws_id=>"snap-aaa-01-01"},
        {:aws_volume_id=>"vol-aaa-01", :aws_id=>"snap-aaa-01-02"}
      ],                              
      "vol-aaa-02"=> [                
        {:aws_volume_id=>"vol-aaa-02", :aws_id=>"snap-aaa-02-01"}
      ]
    }

    assert_equal(expect, grouping_snapshots_by_volume_id(snapshots))
  end

  def test_detect_aged_snapshots
    grouped_snapshots = {
      "vol-aaa-01"=> [
        {:aws_id=>"snap-aaa-01-04", :aws_started_at=>"2012-08-23T09:07:06.000Z"},
        {:aws_id=>"snap-aaa-01-03", :aws_started_at=>"2012-08-22T09:07:06.000Z"},
        {:aws_id=>"snap-aaa-01-02", :aws_started_at=>"2012-08-21T09:07:06.000Z"},
        {:aws_id=>"snap-aaa-01-01", :aws_started_at=>"2012-08-20T09:07:06.000Z"},
      ],                              
      "vol-aaa-02"=> [                
        {:aws_id=>"snap-aaa-02-03", :aws_started_at=>"2012-08-23T09:07:06.000Z"},
        {:aws_id=>"snap-aaa-02-02", :aws_started_at=>"2012-08-22T09:07:06.000Z"},
        {:aws_id=>"snap-aaa-02-01", :aws_started_at=>"2012-12-21T09:07:06.000Z"},
      ],                              
      "vol-aaa-03"=> [                
        {:aws_id=>"snap-aaa-03-03", :aws_started_at=>"2012-08-23T09:07:06.000Z"},
        {:aws_id=>"snap-aaa-03-02", :aws_started_at=>"2012-08-22T09:07:06.000Z"},
        {:aws_id=>"snap-aaa-03-01", :aws_started_at=>"2012-08-21T09:07:06.000Z"},
      ]
    }

    volumes = [
      {:aws_id=>"vol-aaa-01", :tags=>{}                          }, 
      {:aws_id=>"vol-aaa-02", :tags=>{"AutoBackupRetention"=>"2"}},
      {:aws_id=>"vol-aaa-03", :tags=>{"AutoBackupRetention"=>"4"}}
    ]

    expect = ["snap-aaa-01-01", "snap-aaa-02-02"]

    assert_equal(expect, detect_aged_snapshots(grouped_snapshots, volumes))
  end

  def test_detect_deleted_volumes
    grouped_snapshots = {
      "vol-aaa-01"=> [{:aws_id=>"snap-aaa-01-04"}],                          
      "vol-aaa-02"=> [{:aws_id=>"snap-aaa-02-03"}],                          
      "vol-aaa-03"=> [{:aws_id=>"snap-aaa-03-03"}]
    }

    volumes = [
      {:aws_id=>"vol-aaa-03"}
    ]

    expect = ["vol-aaa-01", "vol-aaa-02"]
    assert_equal(expect, detect_deleted_volumes(grouped_snapshots, volumes))
  end
                      
  def test_detect_unused_volumes
    volumes = [
      {:aws_id=>"vol-aaa-01", :aws_status=>"available"},
      {:aws_id=>"vol-aaa-02", :aws_status=>"in-use"},
      {:aws_id=>"vol-aaa-03", :aws_status=>"available"},
      {:aws_id=>"vol-aaa-04", :aws_status=>"in-use"},
    ]
    expect = ["vol-aaa-01", "vol-aaa-03"]
    assert_equal(expect, detect_unused_volumes(volumes))
  end
end

Usage:
$ ruby manage_backup.rb

Run Test:
$ ruby test_libs.rb

@s-tajima Hi!