Skip to content

Instantly share code, notes, and snippets.

@erochest
Created November 14, 2012 21:41
Show Gist options
  • Save erochest/4075023 to your computer and use it in GitHub Desktop.
Save erochest/4075023 to your computer and use it in GitHub Desktop.
Look at rake dumps:all. It's meant to work with Vagrant, but should be easy to move off that.
require 'etc'
require 'fileutils'
require 'peach'
require 'vagrant'
DUMP_DIR = 'dumps/todo'
GIS_ARCHIVE = 'data/geoserver1-20110908-1547.tar.gz'
ROLE_FILE = 'roles'
GEOSERVER_DIR = '/var/lib/tomcat6/webapps/geoserver'
TABLES = %w{
Afghanistan
Alb10contour4ft
Albemarle
Albemarle2010
Alexandria
Arlington
Basemap1
Bronx
Census2000
Charlottesville
Charlottesville2010
Chesapeake
Chesapeake_Bay_Program
Cultural_Landscape_Survey
F_Dists
Fairfax
Falmouth
Goochland
Hanover
Henrico
Loudon
Loudoun
MLBS_2009_vector
NEWBERRY
Norfolk
PapuaNG
Portsmouth
Precincts
RappRiverBasin
Richmond
SAfrica
TJPlanDist
USGS_va_geology
USGSindexmaps
UVA
VA_Counties
VA_Historical_Counties
VA_Political_Districts
VA_Roads
VAgnis
VBMP
Venice_Italy
Virginia_Beach
Washington
Washington_DC
WendyHsu
magic-lantern
providence
uva94
}
task :default => :usage
task :usage do
puts "You forgot to tell the computer what to do; try one of these commands:"
system("rake -T")
end
# Execute a command in the primary VM and write its output to the screen.
def vm_ssh(env, cmd)
puts ">>> '#{cmd}'"
env.primary_vm.channel.execute cmd do |channel, data|
out = $stdout
if channel == :stderr
out = $stderr
end
out.write(data)
out.flush()
end
end
def createuser(env, username, passwd)
vm_ssh(env, "sudo su root <<SCRIPT
sudo -u postgres createuser --superuser --createdb --createrole --pwprompt --echo #{username} << EOF
#{passwd}
#{passwd}
EOF
SCRIPT")
end
def pg_restore(dbname)
filename = "#{DUMP_DIR}/#{dbname}.dump"
vm_dump_file = "/vagrant/#{DUMP_DIR}/#{dbname}.dump"
log_file = "/vagrant/#{DUMP_DIR}/#{dbname}.log"
puts "Restoring/Upgrading #{filename} (#{vm_dump_file}) => #{dbname}"
start = Time.now
env = Vagrant::Environment.new
puts "logging to #{log_file}"
vm_ssh(env, "sh /usr/share/postgresql-9.1-postgis/utils/postgis_restore.pl /usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql #{dbname} #{vm_dump_file} -E=UNICODE > #{log_file}")
vm_ssh(env, "grep ^KEEPING #{log_file}")
# May need to re-create the spatial_ref_sys table:
#
# psql db_name
# truncate spatial_ref_sys;
# \i /usr/share/postgresql/9.1/contrib/postgis-1.5/spatial_ref_sys.sql
# \q
#
# If there are customizations, they'll need to be re-added separately.
puts "mv #{filename} dumps/done"
FileUtils.mv(filename, 'dumps/done')
done = Time.now
puts "[#{dbname}] DONE. Elapsed: #{(done - start) * 1000.0} sec."
puts
end
namespace :pg do
desc 'This configures PostgreSQL for remote logins and users.'
task :config do
puts 'Configuring PostgreSQL.'
env = Vagrant::Environment.new
vm_ssh(env, %{echo "listen_addresses = '*'" | sudo tee --append /etc/postgresql/9.1/main/postgresql.conf})
vm_ssh(env, %{echo "local all all password" | sudo tee --append /etc/postgresql/9.1/main/pg_hba.conf})
vm_ssh(env, %{echo "hostssl all all 0.0.0.0/0 password" | sudo tee --append /etc/postgresql/9.1/main/pg_hba.conf})
Rake::Task['pg:restart'].invoke
end
desc 'This restarts PostgreSQL.'
task :restart do
puts 'Restarting PostgreSQL.'
env = Vagrant::Environment.new
vm_ssh(env, 'sudo /etc/init.d/postgresql restart')
end
desc 'This creates the vagrant PostgreSQL user.'
task :createuser do
puts 'Creating vagrant PostgreSQL user.'
env = Vagrant::Environment.new
createuser(env, 'vagrant', 'vagrant')
createuser(env, 'postgis', 'postgis')
createuser(env, 'geoserver', 'geoserver')
end
desc "This creates all the roles listed in #{ROLE_FILE}, if it exists."
task :createroles do
if File.exists?(ROLE_FILE)
roles = File.readlines(ROLE_FILE).map { |l| l.strip }
sql = roles.map { |r| %(CREATE ROLE "#{r}";\n) }
sh <<COMMAND
psql --host=geoserver2.dev --username=vagrant postgres << EOF
#{sql.join}
EOF
COMMAND
else
puts "#{ROLE_FILE} does not exist. Skipping."
end
end
desc 'This initializes PostGIS on the the VM.'
task :init => ['pg:config',
'pg:createuser',
'pg:createroles']
end
def pg_dump(db_name)
puts "DUMPING #{db_name}"
sh %{pg_dump --host=lon.lib.virginia.edu -Fc #{db_name} > #{DUMP_DIR}/#{db_name}.dump}
puts "DONE #{db_name}"
end
namespace :dumps do
desc "This resets all the dump files by moving them back to #{DUMP_DIR}."
task :reset do
Dir['dumps/done/*'].each do |filename|
puts "mv #{filename} #{DUMP_DIR}"
FileUtils.mv(filename, DUMP_DIR)
end
end
desc "This creates a dump file using pg_dump and saves it to #{DUMP_DIR}."
task :dump, [:dbname] do |t, args|
db_name = args[:dbname]
if db_name.nil?
puts "You must supply a database to dump."
else
pg_dump dbname
end
end
desc "This dumps all tables to #{DUMP_DIR} using n threads (defaults to 4)."
task :all, [:threads] do |t, args|
thread_count = (args[:threads] || '4').to_i
# First, clear out what's there.
puts "rm #{DUMP_DIR}/*"
FileUtils.rm Dir["#{DUMP_DIR}/*"]
# Now import the tables in a thread pool.
puts "Running pg_dump in #{thread_count} threads."
TABLES.peach(thread_count) { |table| pg_dump(table) }
end
end
namespace :tomcat do
desc 'This restarts Tomcat 6.'
task :restart do
puts 'Restarting Tomcat 6.'
env = Vagrant::Environment.new
vm_ssh(env, 'sudo /etc/init.d/tomcat6 restart')
end
desc 'This rests the geoserver web app by removing the directory -- which
forces Tomcat to re-expand the WAR file -- and restarting Tomcat.'
task :war do
puts 'Removing current web app.'
env = Vagrant::Environment.new
webapps = '/var/lib/tomcat6/webapps/'
vm_ssh(env, "[ -d #{webapps}/geoserver ] && sudo rm -rf #{webapps}/geoserver")
Rake::Task['tomcat:restart'].invoke
end
end
namespace :gis do
desc 'Symlink to cover that PostgreSQL extension function libraries are in
different places. This is no longer necessary, if the databases are converted
correctly.'
task :symlink do
env = Vagrant::Environment.new
vm_ssh(env, '[ -d /usr/local/postgis/lib ] || sudo mkdir -p /usr/local/postgis')
vm_ssh(env, '[ -d /usr/local/postgis/lib ] || sudo ln -s /usr/lib/postgresql/9.1/lib /usr/local/postgis/lib')
end
desc 'Restore all the database dumps in DUMP_DIR.
You can specify the host, the port, and the user.'
task :postgis do
puts "Loading data from #{DUMP_DIR} into PostGIS."
Dir[File.join(DUMP_DIR, '*.dump')].each do |filename|
dbname = File.basename(filename, '.dump')
pg_restore(dbname)
sleep(10)
end
end
desc 'Restore for one dump file. dbname should also exist as DUMP_DIR/dbname.dump.'
task :restore, [:dbname] do |t, args|
dbname = args[:dbname]
if dbname.nil?
puts 'No value for dbname.'
else
pg_restore(dbname)
end
end
desc 'Restore N dump files from DUMP_DIR.'
task :restoren, [:n] do |t, args|
n = (args[:n] || '10').to_i
Dir[File.join(DUMP_DIR, '*.dump')].take(n).each do |filename|
dbname = File.basename(filename, '.dump')
pg_restore(dbname)
sleep(10)
end
end
desc 'Clobber the GeoServer web.xml with one that sets the data directory.'
task :webxml do
puts 'Copying web.xml file to geoserver directory.'
env = Vagrant::Environment.new
vm_ssh(env, 'sudo cp /vagrant/etc/web.xml /var/lib/tomcat6/webapps/geoserver/WEB-INF/web.xml')
Rake::Task['tomcat:restart'].invoke
end
desc 'Reset the data files.'
task :datadir, :archive do |t, args|
args.with_defaults(:archive => GIS_ARCHIVE)
if File.exists?(args[:archive])
puts 'Expanding archive files.'
sh %{tar xfz #{args[:archive]}}
FileUtils.rmtree 'geoserver', :verbose => true if File.directory?('geoserver')
FileUtils.mv('geoserver1', 'geoserver')
env = Vagrant::Environment.new
vm_ssh(env, "sudo rm -rf #{GEOSERVER_DIR}/data")
vm_ssh(env, "sudo cp -r /vagrant/geoserver/{data,coverages} #{GEOSERVER_DIR}/")
vm_ssh(env, "sudo chown -R tomcat6.tomcat6 #{GEOSERVER_DIR}/{data,coverages}")
else
puts "Archive file #{args[:archive]} does not exist. Skipping."
end
end
desc 'This installs the GIS fonts.'
task :fonts do
if File.exists?('fonts.tar.gz')
FileUtils.rmtree 'fonts', :verbose => true if File.directory?('fonts')
sh %{tar xfz fonts.tar.gz}
env = Vagrant::Environment.new
vm_ssh(env, 'sudo cp -r /vagrant/fonts/esri /usr/share/fonts/truetype/esri')
vm_ssh(env, 'sudo fc-cache -f -v')
else
puts 'Font archive (fonts.tar.gz) does not exist. Skipping.'
end
end
desc 'This resets the data in the GIS system.'
task :init, [:archive] => ['gis:symlink',
'gis:postgis',
'gis:datadir',
'gis:fonts']
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment