Capture.scala

import kafka.api.FetchRequest
import kafka.consumer.SimpleConsumer
import kafka.message._
import kafka.utils.Utils
import kafka.api.OffsetRequest
import java.io.{File, FileOutputStream}
import com.yammer.metrics.Instrumented
import com.yammer.metrics.reporting.ConsoleReporter
import java.util.concurrent.TimeUnit


val topicName = "example"
val consumer = new SimpleConsumer("localhost", 9092, 10000, 1024 * 1024 * 10)
//val wChannel = new FileOutputStream(new File("out.bin"), true).getChannel()
var offset = consumer.getOffsetsBefore(topicName, 0, OffsetRequest.EARLIEST_TIME, 1).last

object Consumer extends Instrumented {
  val throughputMeter = metrics.meter("received", "megabytes")
  ConsoleReporter.enable(1, TimeUnit.SECONDS)

  def run() {
    while (true) {
      val fetchRequest = new FetchRequest(topicName, 0, offset, 1024 * 1024 * 10)
      val messages = consumer.fetch(fetchRequest)
      for (message <- messages) {
        //println(message.payload.remaining)
        throughputMeter.mark(1)
        //wChannel.write(message.payload)
        offset += MessageSet.entrySize(message)
      }
    }
  }
}

Consumer.run

Consumer.scala output.txt

cscotta@believe:~/Desktop/throughput-test$ scala -cp ~/projects/streaker/target/streaker-0.1.jar Capture.scala 
log4j:WARN No appenders could be found for logger (kafka.consumer.SimpleConsumer).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
9/17/11 7:56:42 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 360
         mean rate = 353.26 megabytes/s
     1-minute rate = 0.00 megabytes/s
     5-minute rate = 0.00 megabytes/s
    15-minute rate = 0.00 megabytes/s



9/17/11 7:56:43 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 747
         mean rate = 373.10 megabytes/s
     1-minute rate = 0.00 megabytes/s
     5-minute rate = 0.00 megabytes/s
    15-minute rate = 0.00 megabytes/s



9/17/11 7:56:44 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 1134
         mean rate = 377.74 megabytes/s
     1-minute rate = 0.00 megabytes/s
     5-minute rate = 0.00 megabytes/s
    15-minute rate = 0.00 megabytes/s



9/17/11 7:56:45 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 1532
         mean rate = 383.31 megabytes/s
     1-minute rate = 0.00 megabytes/s
     5-minute rate = 0.00 megabytes/s
    15-minute rate = 0.00 megabytes/s



9/17/11 7:56:46 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 1921
         mean rate = 384.03 megabytes/s
     1-minute rate = 384.20 megabytes/s
     5-minute rate = 384.20 megabytes/s
    15-minute rate = 384.20 megabytes/s



9/17/11 7:56:47 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 2308
         mean rate = 384.53 megabytes/s
     1-minute rate = 384.20 megabytes/s
     5-minute rate = 384.20 megabytes/s
    15-minute rate = 384.20 megabytes/s



9/17/11 7:56:48 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 2695
         mean rate = 384.89 megabytes/s
     1-minute rate = 384.20 megabytes/s
     5-minute rate = 384.20 megabytes/s
    15-minute rate = 384.20 megabytes/s



9/17/11 7:56:49 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 3035
         mean rate = 379.28 megabytes/s
     1-minute rate = 384.20 megabytes/s
     5-minute rate = 384.20 megabytes/s
    15-minute rate = 384.20 megabytes/s



9/17/11 7:56:50 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 3312
         mean rate = 367.91 megabytes/s
     1-minute rate = 384.20 megabytes/s
     5-minute rate = 384.20 megabytes/s
    15-minute rate = 384.20 megabytes/s



9/17/11 7:56:51 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 3702
         mean rate = 370.12 megabytes/s
     1-minute rate = 381.96 megabytes/s
     5-minute rate = 383.74 megabytes/s
    15-minute rate = 384.04 megabytes/s



9/17/11 7:56:52 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 4099
         mean rate = 372.57 megabytes/s
     1-minute rate = 381.96 megabytes/s
     5-minute rate = 383.74 megabytes/s
    15-minute rate = 384.04 megabytes/s



9/17/11 7:56:53 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 4486
         mean rate = 373.77 megabytes/s
     1-minute rate = 381.96 megabytes/s
     5-minute rate = 383.74 megabytes/s
    15-minute rate = 384.04 megabytes/s



9/17/11 7:56:54 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 4873
         mean rate = 374.79 megabytes/s
     1-minute rate = 381.96 megabytes/s
     5-minute rate = 383.74 megabytes/s
    15-minute rate = 384.04 megabytes/s



9/17/11 7:56:55 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 5263
         mean rate = 375.88 megabytes/s
     1-minute rate = 381.96 megabytes/s
     5-minute rate = 383.74 megabytes/s
    15-minute rate = 384.04 megabytes/s



9/17/11 7:56:56 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 5610
         mean rate = 373.95 megabytes/s
     1-minute rate = 381.93 megabytes/s
     5-minute rate = 383.70 megabytes/s
    15-minute rate = 384.03 megabytes/s



9/17/11 7:56:57 PM =============================================================
Main..anon.1.Consumer:
  received:
             count = 5967
         mean rate = 372.88 megabytes/s
     1-minute rate = 381.93 megabytes/s
     5-minute rate = 383.70 megabytes/s
    15-minute rate = 384.03 megabytes/s

Producer.scala

import kafka.message._
import kafka.producer._
import java.nio.ByteBuffer
import java.util.Properties
import java.util.concurrent.TimeUnit
import com.yammer.metrics.Instrumented
import com.yammer.metrics.reporting.ConsoleReporter


val topicName = "example"
val producerProps = new Properties()
producerProps.put("broker.list", "0:localhost:9092")

val producer = new Producer[String, Message](new ProducerConfig(producerProps))
val messageSize = 1 * 1000 * 1000

val bytes = new Array[Byte](messageSize)

object Producer extends Instrumented {
  val throughputMeter = metrics.meter("sent", "megabytes")
  ConsoleReporter.enable(1, TimeUnit.SECONDS)

  def run() {
    while (true) {
      producer.send(new ProducerData[String, Message](topicName, new Message(bytes)))
      throughputMeter.mark(1)
    }
  }
}

Producer.run

Producer.scala output.txt

cscotta@believe:~/Desktop/throughput-test$ scala -cp ~/projects/streaker/target/streaker-0.1.jar Producer.scala 
log4j:WARN No appenders could be found for logger (kafka.producer.SyncProducer).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
9/17/11 7:56:14 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 188
         mean rate = 184.76 megabytes/s
     1-minute rate = 0.00 megabytes/s
     5-minute rate = 0.00 megabytes/s
    15-minute rate = 0.00 megabytes/s



9/17/11 7:56:15 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 386
         mean rate = 192.80 megabytes/s
     1-minute rate = 0.00 megabytes/s
     5-minute rate = 0.00 megabytes/s
    15-minute rate = 0.00 megabytes/s



9/17/11 7:56:16 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 589
         mean rate = 196.20 megabytes/s
     1-minute rate = 0.00 megabytes/s
     5-minute rate = 0.00 megabytes/s
    15-minute rate = 0.00 megabytes/s



9/17/11 7:56:17 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 791
         mean rate = 197.65 megabytes/s
     1-minute rate = 0.00 megabytes/s
     5-minute rate = 0.00 megabytes/s
    15-minute rate = 0.00 megabytes/s



9/17/11 7:56:18 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 995
         mean rate = 198.92 megabytes/s
     1-minute rate = 199.00 megabytes/s
     5-minute rate = 199.00 megabytes/s
    15-minute rate = 199.00 megabytes/s



9/17/11 7:56:19 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 1199
         mean rate = 199.77 megabytes/s
     1-minute rate = 199.00 megabytes/s
     5-minute rate = 199.00 megabytes/s
    15-minute rate = 199.00 megabytes/s



9/17/11 7:56:20 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 1402
         mean rate = 200.23 megabytes/s
     1-minute rate = 199.00 megabytes/s
     5-minute rate = 199.00 megabytes/s
    15-minute rate = 199.00 megabytes/s



9/17/11 7:56:21 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 1604
         mean rate = 200.45 megabytes/s
     1-minute rate = 199.00 megabytes/s
     5-minute rate = 199.00 megabytes/s
    15-minute rate = 199.00 megabytes/s



9/17/11 7:56:22 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 1808
         mean rate = 200.84 megabytes/s
     1-minute rate = 199.00 megabytes/s
     5-minute rate = 199.00 megabytes/s
    15-minute rate = 199.00 megabytes/s



9/17/11 7:56:23 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 2008
         mean rate = 200.76 megabytes/s
     1-minute rate = 199.29 megabytes/s
     5-minute rate = 199.06 megabytes/s
    15-minute rate = 199.02 megabytes/s



9/17/11 7:56:24 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 2209
         mean rate = 200.78 megabytes/s
     1-minute rate = 199.29 megabytes/s
     5-minute rate = 199.06 megabytes/s
    15-minute rate = 199.02 megabytes/s



9/17/11 7:56:25 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 2410
         mean rate = 200.80 megabytes/s
     1-minute rate = 199.29 megabytes/s
     5-minute rate = 199.06 megabytes/s
    15-minute rate = 199.02 megabytes/s



9/17/11 7:56:26 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 2608
         mean rate = 200.58 megabytes/s
     1-minute rate = 199.29 megabytes/s
     5-minute rate = 199.06 megabytes/s
    15-minute rate = 199.02 megabytes/s



9/17/11 7:56:27 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 2649
         mean rate = 189.18 megabytes/s
     1-minute rate = 199.29 megabytes/s
     5-minute rate = 199.06 megabytes/s
    15-minute rate = 199.02 megabytes/s



9/17/11 7:56:28 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 2841
         mean rate = 189.38 megabytes/s
     1-minute rate = 196.66 megabytes/s
     5-minute rate = 198.52 megabytes/s
    15-minute rate = 198.84 megabytes/s



9/17/11 7:56:29 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 3041
         mean rate = 190.03 megabytes/s
     1-minute rate = 196.66 megabytes/s
     5-minute rate = 198.52 megabytes/s
    15-minute rate = 198.84 megabytes/s



9/17/11 7:56:30 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 3239
         mean rate = 190.51 megabytes/s
     1-minute rate = 196.66 megabytes/s
     5-minute rate = 198.52 megabytes/s
    15-minute rate = 198.84 megabytes/s



9/17/11 7:56:31 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 3437
         mean rate = 190.92 megabytes/s
     1-minute rate = 196.66 megabytes/s
     5-minute rate = 198.52 megabytes/s
    15-minute rate = 198.84 megabytes/s



9/17/11 7:56:32 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 3636
         mean rate = 191.35 megabytes/s
     1-minute rate = 196.66 megabytes/s
     5-minute rate = 198.52 megabytes/s
    15-minute rate = 198.84 megabytes/s



9/17/11 7:56:33 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 3837
         mean rate = 191.83 megabytes/s
     1-minute rate = 196.88 megabytes/s
     5-minute rate = 198.53 megabytes/s
    15-minute rate = 198.84 megabytes/s



9/17/11 7:56:34 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 4034
         mean rate = 192.12 megabytes/s
     1-minute rate = 196.88 megabytes/s
     5-minute rate = 198.53 megabytes/s
    15-minute rate = 198.84 megabytes/s



9/17/11 7:56:35 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 4218
         mean rate = 191.71 megabytes/s
     1-minute rate = 196.88 megabytes/s
     5-minute rate = 198.53 megabytes/s
    15-minute rate = 198.84 megabytes/s



9/17/11 7:56:36 PM =============================================================
Main..anon.1.Producer:
  sent:
             count = 4385
         mean rate = 190.63 megabytes/s
     1-minute rate = 196.88 megabytes/s
     5-minute rate = 198.53 megabytes/s
    15-minute rate = 198.84 megabytes/s