Skip to content

Instantly share code, notes, and snippets.

@palexander
Last active December 28, 2015 11:19
Show Gist options
  • Save palexander/7492687 to your computer and use it in GitHub Desktop.
Save palexander/7492687 to your computer and use it in GitHub Desktop.
Setup for running docker
# Rapper and Java
sudo apt-get install openjdk-6-jre-headless raptor2-utils libxml2-dev
# Get ruby-build dependencies
sudo apt-get install -y git-core curl wget libssl1.0.0 python-yaml build-essential libssl-dev
# Get ruby-build
git clone https://github.com/sstephenson/ruby-build.git
# Install ruby-build and ruby 2.0.0p247
cd ruby-build && sudo ./install.sh
mkdir ~/rubies
/usr/local/bin/ruby-build 2.0.0-p247 ~/rubies/2.0.0-p247
# Add to .profile
export PATH=$HOME/rubies/2.0.0-p247/bin:$PATH
# Install bundler system-wide
gem install bundler
# Pull docker containers
docker pull palexander/fourstore
docker pull palexander/redis
docker pull palexander/solr
namespace :docker do
desc "Run tasks in parallel using docker"
task :test do
services = %w(fourstore redis solr)
internal_ports = {"fourstore" => 8080, "redis" => 6379, "solr" => "8983"}
forks = 8
begin
require 'ontologies_linked_data'
require 'ncbo_annotator'
test_files = FileList['test/**/test*.rb'].shuffle
test_files_sliced = test_files.each_slice(test_files.length / forks).to_a
pids = []
forks.times do |i|
pids << fork do
# Start redis, solr, and 4store
ports = {}
services.each do |srv|
started = false
attempts = 0
until started || attempts > 10
`docker kill #{srv}#{i} && docker rm #{srv}#{i}`
`docker run -t -d -p #{internal_ports[srv]} -name #{srv}#{i} palexander/#{srv}`
started = $?.success?
attempts += 1
end
ports[srv] = `docker port #{srv}#{i} #{internal_ports[srv]}`.split(":").last.to_i
end
puts "Fork #{i} testing #{test_files_sliced[i].join(", ")}"
# redirect stdout
require 'stringio'
sio = StringIO.new
$stdout = sio
$stderr = sio
LinkedData.config do |config|
config.goo_port = ports["fourstore"]
config.goo_redis_port = ports["redis"]
config.http_redis_port = ports["redis"]
config.search_server_url = "http://localhost:#{ports["solr"]}/solr/"
end
Annotator.config do |config|
config.annotator_redis_port = ports["redis"]
config.mgrep_host = "ncbo-stg-app-20"
config.mgrep_port = 55555
end
$QUEUE_REDIS_PORT = ports["redis"]
require_relative "test/test_case"
# Stop tests from auto-running
class ::AppUnit
@@stop_auto_run = true
end
test_files_sliced[i].each {|f| require_relative f}
MiniTest::Unit.runner.run
# reset stdout
$stdout = STDOUT
sio.rewind
puts "", "", "Fork #{i} completed, output:", sio.read, ""
Kernel.exit! # force the fork to end without running at_exit bindings
end
end
pids.each {|pid| Process.wait(pid)}
rescue Exception => e
puts e.message
puts e.backtrace.join("\n\t")
ensure
puts "\n\nStopping docker containers"
forks.times {|i| services.each {|s| `docker kill #{s}#{i} && docker rm #{s}#{i}`}}
Kernel.exit! # force the process to quit without minitest's autorun (triggered on at_exit)
end
end
end
namespace :docker do
desc "Run tasks in parallel using docker"
task :test do
services = %w(fourstore redis solr)
internal_ports = {"fourstore" => 8080, "redis" => 6379, "solr" => "8983"}
forks = 8
begin
require_relative 'lib/ontologies_linked_data'
test_files = FileList['test/**/test*.rb'].shuffle
test_files_sliced = test_files.each_slice(test_files.length / forks).to_a
pids = []
forks.times do |i|
pids << fork do
# Start redis, solr, and 4store
ports = {}
services.each do |srv|
started = false
attempts = 0
until started || attempts > 10
`docker kill #{srv}#{i} && docker rm #{srv}#{i}`
`docker run -t -d -p #{internal_ports[srv]} -name #{srv}#{i} palexander/#{srv}`
started = $?.success?
attempts += 1
end
ports[srv] = `docker port #{srv}#{i} #{internal_ports[srv]}`.split(":").last.to_i
end
puts "Fork #{i} testing #{test_files_sliced[i].join(", ")}"
# redirect stdout
require 'stringio'
sio = StringIO.new
$stdout = sio
$stderr = sio
LinkedData.config do |config|
config.goo_port = ports["fourstore"]
config.goo_redis_port = ports["redis"]
config.http_redis_port = ports["redis"]
config.search_server_url = "http://localhost:#{ports["solr"]}/solr/"
end
require_relative "test/test_case"
# Stop tests from auto-running
class ::LinkedData::Unit
@@stop_auto_run = true
end
test_files_sliced[i].each {|f| require_relative f}
MiniTest::Unit.runner.run
# reset stdout
$stdout = STDOUT
sio.rewind
puts "", "", "Fork #{i} completed, output:", sio.read, ""
Kernel.exit! # force the fork to end without running at_exit bindings
end
end
pids.each {|pid| Process.wait(pid)}
rescue Exception => e
puts e.message
puts e.backtrace.join("\n\t")
ensure
puts "\n\nStopping docker containers"
forks.times {|i| services.each {|s| `docker kill #{s}#{i} && docker rm #{s}#{i}`}}
Kernel.exit! # force the process to quit without minitest's autorun (triggered on at_exit)
end
end
end
# Base config from Vagrantfile plus additions to support docker
config.vm.provider :virtualbox do |vb|
config.vm.box = BOX_NAME
config.vm.box_url = BOX_URI
vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
# Added for docker
vb.customize ["modifyvm", :id, "--ioapic", "on"] # needed for multicpu support
vb.customize ["modifyvm", :id, "--cpus", "4"] # multiple cpus
vb.customize ["modifyvm", :id, "--memory", "1024"] # Added to increase available memory
config.vm.synced_folder "~/Development", "/srv/development"
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment