Enterprise Chef drbd-backups - improved version

drbd-backups.rb

#!/opt/opscode/embedded/bin/ruby
#
# Author:: Lamont Granquist (<[email protected]>)
# Copyright:: Copyright (c) 2011 Opscode, Inc.
# License:: Apache License, Version 2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

require 'optparse'
require 'pp'

options = {
  :drbd_vg => "opscode",
  :drbd_lv => "drbd",
  :drbd_snapsize => "10%VG",
  :mail_to => "root@localhost",
  :pg_data => "/var/opt/opscode/drbd/data/postgresql_9.2",
  :pg_user => "opscode-pgsql",
  :backup_mp => "/var/opt/opscode/drbd-backups",
  :path_str => "/usr/bin:/bin:/usr/sbin:/sbin",
  :pg_backup_file => nil,
  :data => "/var/opt/opscode/drbd/data"
}

# use these binary helpers from omnibus
CHPST="/opt/opscode/embedded/bin/chpst"
PG_DUMPALL="/opt/opscode/embedded/bin/pg_dumpall"

# use these binaires from the O/S, and rely on PATH ( we add /sbin & /usr/sbin later )
GZIP="gzip"
UMOUNT="umount"
LVDISPLAY="lvdisplay"
LVREMOVE="lvremove"
LVCREATE="lvcreate"
DATE="date"
HOSTNAME="hostname"
MAIL="mail"
MKDIR="mkdir"
MOUNT="mount"
DMSETUP="dmsetup"
TAR="tar -zcf"
# The tr handles possible longline newline splits in the full df output
TYPE="df -T #{options[:data]} > /dev/null 2>&1 && df -T #{options[:data]} | tail -n +2 | tr '\n' ' '| awk '{print $2}'"

optparse = OptionParser.new do |opts|

  opts.banner = "Usage: drbd-backups [options]"

  opts.on( '-m', '--mailto EMAIL', 'E-mail address to send errors to' ) do |email|
    options[:mail_to] = email
  end

  opts.on( '-g', '--vg VOLGROUP', 'Volume Group of Logical Volume to backup' ) do |vg|
    options[:drbd_vg] = vg
  end

  opts.on( '-l', '--lv LOGICALVOL', 'Logical Volume to backup' ) do |lv|
    options[:drbd_lv] = lv
  end

  opts.on( '-p', '--pathstr PATHSTRING', 'PATH string to append to PATH' ) do |pathstr|
    options[:path_str] = pathstr
  end

  opts.on( '-M', '--mountpoint MOUNTPOINT', 'Filesystem path to mount backups on' ) do |mountpoint|
    options[:backup_mp] = mountpoint
  end

  opts.on( '-u', '--postgres-user USERNAME', 'Username to change to in order to take PostgreSQL backups' ) do |user|
    options[:pg_user] = user
  end

  opts.on( '-d', '--postgres-data DIRECTORY', 'Directory containing PostgreSQL data to backup' ) do |dir|
    options[:pg_data] = dir
  end

  opts.on( '-f', '--backup-file PATH', 'Filename to dump compressed PostgresSQL data into' ) do |path|
    options[:pg_backup_file] = path
  end

  opts.on( '-s', '--snap-size SIZE', 'Snapshot volume size. Leave default for production install' ) do |size|
    options[:drbd_snapsize] = size
  end

  opts.on( '-h', '--help', 'Display this help' ) do
    puts opts
    exit
  end
end

optparse.parse!

envstr = ENV['PATH'].split(':')
envstr += options[:path_str].split(':')
ENV['PATH'] = envstr.join(':')

sysname = `#{HOSTNAME}`.chomp

if !system("#{TAR} #{options[:data]}/opc-config-bak.tar.gz /etc/opscode")
    system("#{MAIL} -s 'privatechef /etc/opscode/* backup failed on #{sysname}' #{options[:mail_to]} < /dev/null")
end

if File.directory?(options[:pg_data])
  # we are mounting drbd and can take a PostgreSQL backup
  options[:pg_backup_file] ||= "#{options[:pg_data]}/pgsql-backup.out.gz"
  if !system("#{CHPST} -u #{options[:pg_user]} #{PG_DUMPALL} | #{GZIP} --fast > #{options[:pg_backup_file]}")
    system("#{MAIL} -s 'privatechef postgresql backup failed on #{sysname}' #{options[:mail_to]} < /dev/null")
    # keep going on here, a logical volume snapshot that needs crash recovery is better than nothing...
  end
end

# could check if the mountpoint is mounted and fuser -km, but that gives me heebie-jeebies...
system("#{UMOUNT} -f #{options[:backup_mp]} 2>/dev/null")

#
# see:
#  - https://bugzilla.redhat.com/show_bug.cgi?id=577798
#  - https://bugzilla.redhat.com/show_bug.cgi?id=570359
#  - https://bugzilla.redhat.com/show_bug.cgi?id=638711
#  - https://bugzilla.redhat.com/show_bug.cgi?id=721122
#
# or:
#  - yum remove udisks (will rip out nautilus and some other desktopish file-browser-ey things)
#  - comment out line mentioned in https://bugzilla.redhat.com/show_bug.cgi?id=570359#c10
#

count = 0
while true
  if count > 300
    raise "too many times trying to run lvremove, help me!"
    system("#{MAIL} -s 'privatechef lvremove failed 300 times on #{sysname}' #{options[:mail_to]} < /dev/null")
  end
  failed = false
  `#{LVDISPLAY}`.each_line do |line|
    next unless line =~ /LV Name.*(backup-\d+-\d+-\d+-\d+-\d+-\d+)/
    lv_name = $1
    dev_path = "/dev/#{options[:drbd_vg]}/#{lv_name}"
    cmd = "#{LVREMOVE} -f #{dev_path}"
    puts "running #{cmd}"
    failed = true unless system(cmd)
  end
  break unless failed
  failed = false
  `#{DMSETUP} ls`.each_line do |line|
    next unless line =~ /^(.*-backup-\S*)\s/
    lv_name = $1
    cmd = "#{DMSETUP} remove #{lv_name}"
    puts "running #{cmd}"
    failed = true unless system(cmd)
  end
  # dmsetup likes to claim it worked when it didn't, running it still seems to help
  # break unless failed
  count += 1
end

datestamp=`#{DATE} +%Y-%m-%d-%H-%M-%S`.chomp

system("sync")
if !system("#{LVCREATE} -l #{options[:drbd_snapsize]} -s -n backup-#{datestamp} /dev/#{options[:drbd_vg]}/#{options[:drbd_lv]}")
  system("#{MAIL} -s 'privatechef lvcreate failed on #{sysname}' #{options[:mail_to]} < /dev/null")
  exit -1
end

system("#{MKDIR} -p #{options[:backup_mp]}")

type=`#{TYPE}`.chomp
if !system("#{MOUNT} #{if type != "false" then "-t " << type; end} /dev/#{options[:drbd_vg]}/backup-#{datestamp} #{options[:backup_mp]}")
  system("#{MAIL} -s 'privatechef mounting #{options[:backup_mp]} failed on #{sysname}' #{options[:mail_to]} < /dev/null")
end