iconara

import java.util.*;
import com.rabbitmq.client.*;


public class Consumer {
	private static int count = 0;

	public static void main(String[] args) throws Exception {
		Random random = new Random();

iconara

--- Main.css	2011-07-07 16:17:44.000000000 +0200
+++ Main.css	2011-07-07 16:13:21.000000000 +0200
@@ -117,19 +117,19 @@
 
 #Content {padding: 0 40px 40px 40px;}
 .section {float: left; width: 280px; margin: 0 20px 20px 0;}
-.section h2 {margin: 0 0 20px 0; padding: 23px 25px; color: #fff; background-color: #00FF00;
+.section h2 {margin: 0 0 20px 0; padding: 23px 25px; color: #444; background-color: #00FF00;
 	-moz-border-radius: 5px;
 	-webkit-border-radius: 5px;

iconara

require 'open-uri'
require 'json'


class Queue
  def initialize(info)
    @info = info
  end
  
  def name

iconara

2011-09-21 18:43:41 ZooKeeper [INFO] Client environment:zookeeper.version=3.3.2-1031432, built on 11/05/2010 05:32 GMT
2011-09-21 18:43:41 ZooKeeper [INFO] Client environment:host.name=172.16.160.1
2011-09-21 18:43:41 ZooKeeper [INFO] Client environment:java.version=1.7.0-internal
2011-09-21 18:43:41 ZooKeeper [INFO] Client environment:java.vendor=Oracle Corporation
2011-09-21 18:43:41 ZooKeeper [INFO] Client environment:java.home=/Library/Java/JavaVirtualMachines/1.7.0.jdk/Contents/Home/jre
2011-09-21 18:43:41 ZooKeeper [INFO] Client environment:java.class.path=/Users/theo/Downloads/storm-0.5.0/lib/clj-time-0.3.0.jar:/Users/theo/Downloads/storm-0.5.0/lib/clojure-1.2.0.jar:/Users/theo/Downloads/storm-0.5.0/lib/clojure-contrib-1.2.0.jar:/Users/theo/Downloads/storm-0.5.0/lib/clout-0.4.1.jar:/Users/theo/Downloads/storm-0.5.0/lib/commons-cli-1.2.jar:/Users/theo/Downloads/storm-0.5.0/lib/commons-codec-1.3.jar:/Users/theo/Downloads/storm-0.5.0/lib/commons-el-1.0.jar:/Users/theo/Downloads/storm-0.5.0/lib/commons-exe

iconara

Code/mikka% rvm install jruby-1.6.5
jruby-1.6.5 - #fetching 
jruby-1.6.5 - #downloading jruby-bin-1.6.5, this may take a while depending on your connection...
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 14.6M  100 14.6M    0     0   156k      0  0:01:35  0:01:35 --:--:--  210k
jruby-1.6.5 - #extracting jruby-bin-1.6.5 to /Users/theo/.rvm/src/jruby-1.6.5
jruby-1.6.5 - #extracted to /Users/theo/.rvm/src/jruby-1.6.5
Building Nailgun
jruby-1.6.5 - #installing to /Users/theo/.rvm/rubies/jruby-1.6.5

iconara

EC2 = {
  'm1.small'    => {:ram =>  1.7, :ecus =>  1.0, :cores =>  1, :storage =>  160},
  'm1.large'    => {:ram =>  7.5, :ecus =>  4.0, :cores =>  2, :storage =>  850},
  'm1.xlarge'   => {:ram => 15.0, :ecus =>  8.0, :cores =>  4, :storage => 1690},
  't1.micro'    => {:ram =>  0.6, :ecus =>  2.0, :cores =>  1, :storage =>    0}, # 613/1024 Gb RAM and actually less than 2 CPUs, but bursting up to 2
  'm2.xlarge'   => {:ram => 17.1, :ecus =>  6.5, :cores =>  2, :storage =>  420},
  'm2.2xlarge'  => {:ram => 34.2, :ecus => 13.0, :cores =>  4, :storage =>  850},
  'm2.4xlarge'  => {:ram => 68.4, :ecus => 26.0, :cores =>  8, :storage => 1690},
  'c1.medium'   => {:ram =>  1.7, :ecus =>  5.0, :cores =>  2, :storage =>  350},
  'c1.xlarge'   => {:ram =>  7.0, :ecus => 20.0, :cores =>  8, :storage => 1690},

iconara

# gem install aws-sdk

require 'aws'


PRICE_PER_GB_MONTH = 0.11

ACCESS_KEY = '...'
SECRET_KEY = '...'
ENDPOINT = 'eu-west-1.ec2.amazonaws.com'

iconara

# encoding: utf-8

require 'java'


module JUC
  java_import 'java.util.concurrent.Executors'
  java_import 'java.util.concurrent.TimeUnit'
end

iconara

Theo Hultberg

Contact details

• Email: [email protected]
• Twitter: @iconara
• Website or Blog: http://architecturalatrocities.com
• Company: Burt

Speaker bio

iconara

At Burt we're heavy into RabbitMQ, we use it as a series of tubes that transports almost all of the wonderful data around our platform. It's served us well, apart from a tiny, tiny detail: give it too much of the good stuff and it sort of gets drunk and won't have anymore. Legend has it that version 2.8 solves that problem, and as far as we can tell it's almost all true. Read on for pretty graphs.

[[MORE]]

Background

Our whole platform runs off of Amazon EC2, and for this test set we set up RabbitMQ clusters of 1, 2 and 3 c1.xlarge instances.

Each cluster gets 24 durable queues, we basically use RabbitMQ for transport, and to get the maximum throughput you want to have many queues, since each queue is essentially single threaded.

We loaded the queues by parsing about 2.3 gigs worth of messages from old production logs. The messages were combined and sent in batches of ten (another good technique for getting higher throughput), and routed explicitly to one of then 24 queues based on a property o