gistfile1.txt

2016-04-07T02:32:36.223Z ERROR <> [worker-0] com.opentable.logging.KafkaPipeline - [default] pipeline dying due to exception
java.lang.IllegalStateException: Memory records is not writable
at org.apache.kafka.common.record.MemoryRecords.append(MemoryRecords.java:93)
at org.apache.kafka.clients.producer.internals.RecordBatch.tryAppend(RecordBatch.java:69)
at org.apache.kafka.clients.producer.internals.RecordAccumulator.append(RecordAccumulator.java:168)
at org.apache.kafka.clients.producer.KafkaProducer.send(KafkaProducer.java:435)
at com.opentable.logging.KafkaPipeline.writeOut(KafkaPipeline.java:216)
at com.opentable.logging.KafkaPipeline.processMessage(KafkaPipeline.java:186)
at java.lang.Iterable.forEach(Iterable.java:75)
at com.opentable.logging.KafkaPipeline.copyQueue(KafkaPipeline.java:145)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
2016-04-07T02:32:41.901Z INFO <> [worker-1] stats - [default] INPUT:815.96 OUT:31.32 ACK:31.33 DLQ:0.00 DROP:784.63 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:33:11.902Z INFO <> [worker-1] stats - [default] INPUT:738.31 OUT:28.34 ACK:28.35 DLQ:0.00 DROP:709.97 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:33:41.901Z INFO <> [worker-1] stats - [default] INPUT:668.05 OUT:25.64 ACK:25.65 DLQ:0.00 DROP:642.40 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:34:11.901Z INFO <> [worker-1] stats - [default] INPUT:604.48 OUT:23.20 ACK:23.21 DLQ:0.00 DROP:581.27 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:34:41.901Z INFO <> [worker-1] stats - [default] INPUT:546.95 OUT:20.99 ACK:21.00 DLQ:0.00 DROP:525.96 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:34:45.988Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.client.ClientUtils$ - Fetching metadata from broker BrokerEndPoint(0,logging-kafka-qa-sf-1.otenv.com,9092) with correla
2016-04-07T02:34:46.131Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Connected to logging-kafka-qa-sf-1.otenv.com:9092 for producing
2016-04-07T02:34:46.217Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from logging-kafka-qa-sf-1.otenv.com:9092
2016-04-07T02:34:46.219Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 4 leader: none replicas: BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092),BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,4]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:46.219Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 7 leader: none replicas: BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092),BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,7]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:46.220Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 3 leader: none replicas: BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092),BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,3]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:46.220Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 0 leader: none replicas: BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092),BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,0]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:46.220Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-12.qasql.opentable.com:9092
2016-04-07T02:34:46.221Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-11.qasql.opentable.com:9092
2016-04-07T02:34:46.225Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Failed to send data since partitions [logstash,0] don't have a leader
2016-04-07T02:34:46.237Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Back off for 100 ms before retrying send. Remaining retries = 3
2016-04-07T02:34:46.340Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.client.ClientUtils$ - Fetching metadata from broker BrokerEndPoint(0,logging-kafka-qa-sf-1.otenv.com,9092) with correlation id 387 for 1 topic(s) Set(logstash)
2016-04-07T02:34:46.341Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Connected to logging-kafka-qa-sf-1.otenv.com:9092 for producing
2016-04-07T02:34:46.351Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from logging-kafka-qa-sf-1.otenv.com:9092
2016-04-07T02:34:46.352Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 4 leader: none replicas: BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092),BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,4]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:46.352Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 7 leader: none replicas: BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092),BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,7]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:46.353Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 3 leader: none replicas: BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092),BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,3]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:46.354Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 0 leader: none replicas: BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092),BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,0]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:46.357Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-12.qasql.opentable.com:9092
2016-04-07T02:34:46.358Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-11.qasql.opentable.com:9092
2016-04-07T02:34:46.366Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Failed to send data since partitions [logstash,0] don't have a leader
2016-04-07T02:34:46.367Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Back off for 100 ms before retrying send. Remaining retries = 2
2016-04-07T02:34:46.502Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.client.ClientUtils$ - Fetching metadata from broker BrokerEndPoint(2,logging-kafka-qa-sf-3.otenv.com,9092) with correlation id 389 for 1 topic(s) Set(logstash)
2016-04-07T02:34:47.507Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Connected to logging-kafka-qa-sf-3.otenv.com:9092 for producing
2016-04-07T02:34:47.512Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from logging-kafka-qa-sf-3.otenv.com:9092
2016-04-07T02:34:47.513Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 4 leader: none replicas: BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092),BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,4]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.515Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 7 leader: none replicas: BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092),BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,7]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.515Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 3 leader: none replicas: BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092),BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,3]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.516Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 0 leader: none replicas: BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092),BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,0]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.516Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-12.qasql.opentable.com:9092
2016-04-07T02:34:47.516Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-11.qasql.opentable.com:9092
2016-04-07T02:34:47.517Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Failed to send data since partitions [logstash,0] don't have a leader
2016-04-07T02:34:47.517Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Back off for 100 ms before retrying send. Remaining retries = 1
2016-04-07T02:34:47.618Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.client.ClientUtils$ - Fetching metadata from broker BrokerEndPoint(2,logging-kafka-qa-sf-3.otenv.com,9092) with correlation id 391 for 1 topic(s) Set(logstash)
2016-04-07T02:34:47.619Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Connected to logging-kafka-qa-sf-3.otenv.com:9092 for producing
2016-04-07T02:34:47.624Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from logging-kafka-qa-sf-3.otenv.com:9092
2016-04-07T02:34:47.624Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 4 leader: none replicas: BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092),BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,4]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.625Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 7 leader: none replicas: BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092),BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,7]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.626Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 3 leader: none replicas: BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092),BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,3]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.628Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 0 leader: none replicas: BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092),BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,0]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.628Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-12.qasql.opentable.com:9092
2016-04-07T02:34:47.628Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-11.qasql.opentable.com:9092
2016-04-07T02:34:47.629Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Failed to send data since partitions [logstash,0] don't have a leader
2016-04-07T02:34:47.629Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Back off for 100 ms before retrying send. Remaining retries = 0
2016-04-07T02:34:47.730Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.client.ClientUtils$ - Fetching metadata from broker BrokerEndPoint(0,logging-kafka-qa-sf-1.otenv.com,9092) with correlation id 393 for 1 topic(s) Set(logstash)
2016-04-07T02:34:47.731Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Connected to logging-kafka-qa-sf-1.otenv.com:9092 for producing
2016-04-07T02:34:47.734Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from logging-kafka-qa-sf-1.otenv.com:9092
2016-04-07T02:34:47.735Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 4 leader: none replicas: BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092),BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,4]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.736Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 7 leader: none replicas: BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092),BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,7]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.736Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 3 leader: none replicas: BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092),BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,3]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.736Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.BrokerPartitionInfo - Error while fetching metadata partition 0 leader: none replicas: BrokerEndPoint(10,qa-sf-logkafka-10.qasql.opentable.com,9092),BrokerEndPoint(13,qa-sf-logkafka-13.qasql.opentable.com,9092) isr: isUnderReplicated: true for topic partition [logstash,0]: [class kafka.common.LeaderNotAvailableException]
2016-04-07T02:34:47.737Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-12.qasql.opentable.com:9092
2016-04-07T02:34:47.737Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-11.qasql.opentable.com:9092
2016-04-07T02:34:47.739Z ERROR <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Failed to send requests for topics logstash with correlation ids in [385,393]
2016-04-07T02:34:47.740Z ERROR <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.ProducerSendThread - Error in handling batch of 1 events
kafka.common.FailedToSendMessageException: Failed to send messages after 3 tries.
at kafka.producer.async.DefaultEventHandler.handle(DefaultEventHandler.scala:91)
at kafka.producer.async.ProducerSendThread.tryToHandle(ProducerSendThread.scala:105)
at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$3.apply(ProducerSendThread.scala:88)
at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$3.apply(ProducerSendThread.scala:68)
at scala.collection.immutable.Stream.foreach(Stream.scala:547)
at kafka.producer.async.ProducerSendThread.processEvents(ProducerSendThread.scala:67)
at kafka.producer.async.ProducerSendThread.run(ProducerSendThread.scala:45)
2016-04-07T02:34:52.743Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Failed to send data since partitions [logstash,0] don't have a leader
2016-04-07T02:34:52.744Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Back off for 100 ms before retrying send. Remaining retries = 3
2016-04-07T02:34:52.845Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.client.ClientUtils$ - Fetching metadata from broker BrokerEndPoint(2,logging-kafka-qa-sf-3.otenv.com,9092) with correlation id 449 for 1 topic(s) Set(logstash)
2016-04-07T02:34:52.846Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Connected to logging-kafka-qa-sf-3.otenv.com:9092 for producing
2016-04-07T02:34:52.848Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from logging-kafka-qa-sf-3.otenv.com:9092
2016-04-07T02:34:52.848Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-13.qasql.opentable.com:9092
2016-04-07T02:34:52.848Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-12.qasql.opentable.com:9092
2016-04-07T02:34:52.848Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-11.qasql.opentable.com:9092
2016-04-07T02:34:52.856Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Connected to qa-sf-logkafka-13.qasql.opentable.com:9092 for producing
2016-04-07T02:35:11.901Z INFO <> [worker-1] stats - [default] INPUT:494.90 OUT:19.00 ACK:19.00 DLQ:0.00 DROP:475.90 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:35:41.901Z INFO <> [worker-1] stats - [default] INPUT:447.81 OUT:17.19 ACK:17.19 DLQ:0.00 DROP:430.62 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:36:11.901Z INFO <> [worker-1] stats - [default] INPUT:405.19 OUT:15.55 ACK:15.56 DLQ:0.00 DROP:389.64 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:36:41.901Z INFO <> [worker-1] stats - [default] INPUT:366.63 OUT:14.07 ACK:14.08 DLQ:0.00 DROP:352.56 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:37:11.901Z INFO <> [worker-1] stats - [default] INPUT:331.74 OUT:12.73 ACK:12.74 DLQ:0.00 DROP:319.01 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:37:41.901Z INFO <> [worker-1] stats - [default] INPUT:300.17 OUT:11.52 ACK:11.53 DLQ:0.00 DROP:288.65 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:38:11.901Z INFO <> [worker-1] stats - [default] INPUT:271.61 OUT:10.43 ACK:10.43 DLQ:0.00 DROP:261.18 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:38:41.901Z INFO <> [worker-1] stats - [default] INPUT:245.76 OUT:9.43 ACK:9.44 DLQ:0.00 DROP:236.33 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:39:11.901Z INFO <> [worker-1] stats - [default] INPUT:222.37 OUT:8.54 ACK:8.54 DLQ:0.00 DROP:213.84 (messages/sec) processing time 75%:10.91 95%:48.71 99%:95.96 (µs/msg)
2016-04-07T02:39:12.877Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Disconnecting from qa-sf-logkafka-13.qasql.opentable.com:9092
2016-04-07T02:39:12.880Z WARN <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Failed to send producer request with correlation id 533 to broker 13 with data for partitions [logstash,0]
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.writev0(Native Method)
at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51)
at sun.nio.ch.IOUtil.write(IOUtil.java:148)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:504)
at java.nio.channels.SocketChannel.write(SocketChannel.java:502)
at org.apache.kafka.common.network.ByteBufferSend.writeTo(ByteBufferSend.java:57)
at kafka.network.RequestOrResponseSend.writeCompletely(RequestOrResponseSend.scala:50)
at kafka.network.BlockingChannel.send(BlockingChannel.scala:113)
at kafka.producer.SyncProducer.liftedTree1$1(SyncProducer.scala:75)
at kafka.producer.SyncProducer.kafka$producer$SyncProducer$$doSend(SyncProducer.scala:74)
at kafka.producer.SyncProducer$$anonfun$send$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SyncProducer.scala:105)
at kafka.producer.SyncProducer$$anonfun$send$1$$anonfun$apply$mcV$sp$1.apply(SyncProducer.scala:105)
at kafka.producer.SyncProducer$$anonfun$send$1$$anonfun$apply$mcV$sp$1.apply(SyncProducer.scala:105)
at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33)
at kafka.producer.SyncProducer$$anonfun$send$1.apply$mcV$sp(SyncProducer.scala:104)
at kafka.producer.SyncProducer$$anonfun$send$1.apply(SyncProducer.scala:104)
at kafka.producer.SyncProducer$$anonfun$send$1.apply(SyncProducer.scala:104)
at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33)
at kafka.producer.SyncProducer.send(SyncProducer.scala:103)
at kafka.producer.async.DefaultEventHandler.kafka$producer$async$DefaultEventHandler$$send(DefaultEventHandler.scala:257)
at kafka.producer.async.DefaultEventHandler$$anonfun$dispatchSerializedData$2.apply(DefaultEventHandler.scala:108)
at kafka.producer.async.DefaultEventHandler$$anonfun$dispatchSerializedData$2.apply(DefaultEventHandler.scala:100)
at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:772)
at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:226)
at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39)
at scala.collection.mutable.HashMap.foreach(HashMap.scala:98)
at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:771)
at kafka.producer.async.DefaultEventHandler.dispatchSerializedData(DefaultEventHandler.scala:100)
at kafka.producer.async.DefaultEventHandler.handle(DefaultEventHandler.scala:73)
at kafka.producer.async.ProducerSendThread.tryToHandle(ProducerSendThread.scala:105)
at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$3.apply(ProducerSendThread.scala:88)
at kafka.producer.async.ProducerSendThread$$anonfun$processEvents$3.apply(ProducerSendThread.scala:68)
at scala.collection.immutable.Stream.foreach(Stream.scala:547)
at kafka.producer.async.ProducerSendThread.processEvents(ProducerSendThread.scala:67)
at kafka.producer.async.ProducerSendThread.run(ProducerSendThread.scala:45)
2016-04-07T02:39:12.880Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] k.producer.async.DefaultEventHandler - Back off for 100 ms before retrying send. Remaining retries = 3
2016-04-07T02:39:12.981Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.client.ClientUtils$ - Fetching metadata from broker BrokerEndPoint(2,logging-kafka-qa-sf-3.otenv.com,9092) with correlation id 534 for 1 topic(s) Set(logstash)
2016-04-07T02:39:12.990Z INFO <> [ProducerSendThread-pp-buzzsaw-1-mesos-slave1-qa-sf.qasql.opentable.com] kafka.producer.SyncProducer - Connected to logging-kafka-qa-sf-3.otenv.com:9092 for producing