development_delayed_jobs_tailing_after.log

Apr 03 13:58:22 le-mbr rails[32401]: SQL (1.6ms)  UPDATE "delayed_jobs" SET locked_at = '2013-04-03 05:58:22.886930', locked_by = 'host:le-mbr pid:32401' WHERE (id = 718125 and (locked_at is null or locked_at < '2013-04-03 01:58:22.886930') and (run_at <= '2013-04-03 05:58:22.886930'))
Apr 03 13:58:22 le-mbr rails[32401]: Relationship Load (0.6ms)  SELECT "relationships".* FROM "relationships" WHERE "relationships"."id" = $1 LIMIT 1  [["id", 63280]]
Apr 03 13:58:22 le-mbr rails[32401]: Controller Exception: OMG TESTIUNG!!!["/Users/lewang/src/website/lib/shared_group_invalidates.rb:22:in `invalidate_shared_group'", "/Users/lewang/src/website/lib/shared_group_invalidates.rb:16:in `invalidate_shared_group_self_without_delay'", "/Users/lewang/src/website/lib/background_airbrake.rb:26:in `send'", "/Users/lewang/src/website/lib/background_airbrake.rb:26:in `perform'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/backend/base.rb:94:in `invoke_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/backend/base.rb:91:in `invoke_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:182:in `run'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/timeout.rb:67:in `timeout'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:182:in `run'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/benchmark.rb:308:in `realtime'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:181:in `run'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:238:in `reserve_and_run_one_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:238:in `reserve_and_run_one_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:166:in `work_off'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:165:in `times'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:165:in `work_off'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:133:in `start'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/benchmark.rb:308:in `realtime'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:132:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:129:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:128:in `loop'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:128:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/plugins/clear_locks.rb:7:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/plugins/clear_locks.rb:7", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `add'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `add'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:127:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/tasks.rb:9", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:246:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:246:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:241:in `each'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:241:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:184:in `invoke_with_call_chain'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/monitor.rb:242:in `synchronize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:177:in `invoke_with_call_chain'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:170:in `invoke'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:143:in `invoke_task'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `top_level'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `each'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `top_level'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:110:in `run_with_threads'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:95:in `top_level'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:73:in `run'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:160:in `standard_exception_handling'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:70:in `run'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/bin/rake:33", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/bin/rake:23:in `load'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/bin/rake:23"]
Apr 03 13:58:22 le-mbr rails[32401]: Controller Exception: OMG TESTIUNG!!!["/Users/lewang/src/website/lib/shared_group_invalidates.rb:22:in `invalidate_shared_group'", "/Users/lewang/src/website/lib/shared_group_invalidates.rb:16:in `invalidate_shared_group_self_without_delay'", "/Users/lewang/src/website/lib/background_airbrake.rb:26:in `send'", "/Users/lewang/src/website/lib/background_airbrake.rb:26:in `perform'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/backend/base.rb:94:in `invoke_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/backend/base.rb:91:in `invoke_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:182:in `run'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/timeout.rb:67:in `timeout'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:182:in `run'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/benchmark.rb:308:in `realtime'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:181:in `run'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:238:in `reserve_and_run_one_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:238:in `reserve_and_run_one_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:166:in `work_off'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:165:in `times'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:165:in `work_off'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:133:in `start'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/benchmark.rb:308:in `realtime'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:132:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:129:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:128:in `loop'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:128:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/plugins/clear_locks.rb:7:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/plugins/clear_locks.rb:7", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `add'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `add'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:127:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/tasks.rb:9", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:246:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:246:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:241:in `each'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:241:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:184:in `invoke_with_call_chain'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/monitor.rb:242:in `synchronize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:177:in `invoke_with_call_chain'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:170:in `invoke'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:143:in `invoke_task'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `top_level'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `each'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `top_level'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:110:in `run_with_threads'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:95:in `top_level'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:73:in `run'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:160:in `standard_exception_handling'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:70:in `run'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/bin/rake:33", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/bin/rake:23:in `load'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/bin/rake:23"]
Apr 03 13:58:22 le-mbr rails[32401]: 2013-04-03T13:58:22+0800: [Worker(host:le-mbr pid:32401)] * [JOB] host:le-mbr pid:32401 failed with StandardError: OMG TESTIUNG!!! - 0 failed attempts
Apr 03 13:58:22 le-mbr rails[32401]: 2013-04-03T13:58:22+0800: [Worker(host:le-mbr pid:32401)] PERMANENTLY removing Relationship#invalidate_shared_group_self_without_delay because of 1 consecutive failures.
Apr 03 13:58:22 le-mbr rails[32401]:  (0.1ms)  BEGIN
Apr 03 13:58:22 le-mbr rails[32401]:  (0.6ms)  UPDATE "delayed_jobs" SET "updated_at" = '2013-04-03 05:58:22.930124', "attempts" = 1, "locked_at" = '2013-04-03 05:58:22.886930', "locked_by" = 'host:le-mbr pid:32401', "failed_at" = '2013-04-03 05:58:22.928601', "last_error" = 'OMG TESTIUNG!!!/Users/lewang/src/website/lib/shared_group_invalidates.rb:22:in `invalidate_shared_group''/Users/lewang/src/website/lib/shared_group_invalidates.rb:16:in `invalidate_shared_group_self_without_delay''/Users/lewang/src/website/lib/background_airbrake.rb:26:in `send''/Users/lewang/src/website/lib/background_airbrake.rb:26:in `perform''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/backend/base.rb:94:in `invoke_job''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/backend/base.rb:91:in `invoke_job''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:182:in `run''/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/timeout.rb:67:in `timeout''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:182:in `run''/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/benchmark.rb:308:in `realtime''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:181:in `run''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:238:in `reserve_and_run_one_job''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:238:in `reserve_and_run_one_job''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:166:in `work_off''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:165:in `times''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:165:in `work_off''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:133:in `start''/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/benchmark.rb:308:in `realtime''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:132:in `start''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:129:in `start''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:128:in `loop''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:128:in `start''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/plugins/clear_locks.rb:7:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/plugins/clear_locks.rb:7/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `add''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `add''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:127:in `start''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/tasks.rb:9/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:246:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:246:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:241:in `each''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:241:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:184:in `invoke_with_call_chain''/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/monitor.rb:242:in `synchronize''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:177:in `invoke_with_call_chain''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:170:in `invoke''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:143:in `invoke_task''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `top_level''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `each''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `top_level''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:110:in `run_with_threads''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:95:in `top_level''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:73:in `run''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:160:in `standard_exception_handling''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:70:in `run''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/bin/rake:33/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/bin/rake:23:in `load''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/bin/rake:23' WHERE "delayed_jobs"."id" = 718125
Apr 03 13:58:22 le-mbr rails[32401]:  (0.3ms)  COMMIT
Apr 03 13:58:22 le-mbr rails[32401]: 2013-04-03T13:58:22+0800: [Worker(host:le-mbr pid:32401)] 1 jobs processed at 19.8922 j/s, 1 failed ...

development_delayed_jobs_tailing_before.log

Apr 03 13:53:36 le-mbr rails[31645]: Started DelayedJobs::SharedGroupInvalidator ???
Apr 03 13:53:36 le-mbr rails[31645]: Params: {"target":{"updated_at":"2013-04-03T13:53:36+08:00","user_id":4014,"id":63280,"company_id":10066,"disabled":true,"role_id":16,"created_at":"2013-02-14T07:03:15+08:00"}}
Apr 03 13:53:36 le-mbr rails[31645]: Completed DelayedJobs::SharedGroupInvalidator ???
Apr 03 13:53:36 le-mbr rails[31645]: Error DelayedJobs::SharedGroupInvalidator ???
Apr 03 13:53:36 le-mbr rails[31645]: Exception OMG TESTIUNG!!!
Apr 03 13:53:36 le-mbr rails[31645]: Controller Exception: OMG TESTIUNG!!!["/Users/lewang/src/website/lib/shared_group_invalidates.rb:20:in `invalidate_shared_group'", "/Users/lewang/src/website/lib/delayed_jobs.rb:837:in `run_job'", "/Users/lewang/src/website/lib/delayed_jobs.rb:46:in `perform'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/oink-0.9.3/lib/oink/middleware.rb:17:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/oink-0.9.3/lib/oink/middleware.rb:17:in `call'", "/Users/lewang/src/website/lib/delayed_jobs.rb:47:in `perform'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/backend/base.rb:94:in `invoke_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/backend/base.rb:91:in `invoke_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:182:in `run'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/timeout.rb:67:in `timeout'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:182:in `run'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/benchmark.rb:308:in `realtime'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:181:in `run'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:238:in `reserve_and_run_one_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:238:in `reserve_and_run_one_job'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:166:in `work_off'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:165:in `times'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:165:in `work_off'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:133:in `start'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/benchmark.rb:308:in `realtime'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:132:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:129:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:128:in `loop'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:128:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/plugins/clear_locks.rb:7:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/plugins/clear_locks.rb:7", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `add'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `add'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:127:in `start'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/tasks.rb:9", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:246:in `call'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:246:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:241:in `each'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:241:in `execute'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:184:in `invoke_with_call_chain'", "/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/monitor.rb:242:in `synchronize'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:177:in `invoke_with_call_chain'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:170:in `invoke'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:143:in `invoke_task'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `top_level'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `each'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `top_level'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:110:in `run_with_threads'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:95:in `top_level'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:73:in `run'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:160:in `standard_exception_handling'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:70:in `run'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/bin/rake:33", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/bin/rake:23:in `load'", "/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/bin/rake:23"]
Apr 03 13:53:36 le-mbr rails[31645]: 2013-04-03T13:53:36+0800: [Worker(host:le-mbr pid:31645)] * [JOB] host:le-mbr pid:31645 failed with StandardError: OMG TESTIUNG!!! - 0 failed attempts
Apr 03 13:53:36 le-mbr rails[31645]: 2013-04-03T13:53:36+0800: [Worker(host:le-mbr pid:31645)] PERMANENTLY removing DelayedJobs::SharedGroupInvalidator because of 1 consecutive failures.
Apr 03 13:53:36 le-mbr rails[31645]: Failure DelayedJobs::SharedGroupInvalidator ???
Apr 03 13:53:36 le-mbr rails[31645]:  (0.1ms)  BEGIN
Apr 03 13:53:36 le-mbr rails[31645]:  (0.9ms)  UPDATE "delayed_jobs" SET "last_error" = 'OMG TESTIUNG!!!/Users/lewang/src/website/lib/shared_group_invalidates.rb:20:in `invalidate_shared_group''/Users/lewang/src/website/lib/delayed_jobs.rb:837:in `run_job''/Users/lewang/src/website/lib/delayed_jobs.rb:46:in `perform''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/oink-0.9.3/lib/oink/middleware.rb:17:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/oink-0.9.3/lib/oink/middleware.rb:17:in `call''/Users/lewang/src/website/lib/delayed_jobs.rb:47:in `perform''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/backend/base.rb:94:in `invoke_job''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/backend/base.rb:91:in `invoke_job''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:182:in `run''/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/timeout.rb:67:in `timeout''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:182:in `run''/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/benchmark.rb:308:in `realtime''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:181:in `run''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:238:in `reserve_and_run_one_job''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:238:in `reserve_and_run_one_job''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:166:in `work_off''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:165:in `times''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:165:in `work_off''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:133:in `start''/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/benchmark.rb:308:in `realtime''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:132:in `start''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:129:in `start''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:128:in `loop''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:128:in `start''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/plugins/clear_locks.rb:7:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/plugins/clear_locks.rb:7/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `add''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:60:in `initialize''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:78:in `add''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:65:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/lifecycle.rb:38:in `run_callbacks''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/worker.rb:127:in `start''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/delayed_job-3.0.3/lib/delayed/tasks.rb:9/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:246:in `call''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:246:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:241:in `each''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:241:in `execute''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:184:in `invoke_with_call_chain''/Users/lewang/.rvm/rubies/ree-1.8.7-2012.02/lib/ruby/1.8/monitor.rb:242:in `synchronize''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:177:in `invoke_with_call_chain''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/task.rb:170:in `invoke''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:143:in `invoke_task''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `top_level''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `each''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:101:in `top_level''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:110:in `run_with_threads''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:95:in `top_level''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:73:in `run''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:160:in `standard_exception_handling''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/lib/rake/application.rb:70:in `run''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/gems/rake-10.0.4/bin/rake:33/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/bin/rake:23:in `load''/Users/lewang/.rvm/gems/ree-1.8.7-2012.02@agworld-3.2.11/bin/rake:23', "locked_at" = '2013-04-03 05:53:36.865786', "updated_at" = '2013-04-03 05:53:36.916928', "attempts" = 1, "failed_at" = '2013-04-03 05:53:36.915502', "locked_by" = 'host:le-mbr pid:31645' WHERE "delayed_jobs"."id" = 718120
Apr 03 13:53:37 le-mbr rails[31645]:  (1.6ms)  COMMIT
Apr 03 13:53:37 le-mbr rails[31645]: 2013-04-03T13:53:37+0800: [Worker(host:le-mbr pid:31645)] 1 jobs processed at 3.4922 j/s, 1 failed ...