Crude notes on setting up a kafka cluster node with embedded file notes.

kafkaCmds.txt

$ wget http://apache.mirrors.tds.net/kafka/0.9.0.0/kafka_2.11-0.9.0.0.tgz
$ apt-get install -y software-properties-common unzip
$ add-apt-repository ppa:openjdk-r/ppa
$ apt-get update
$ apt-get install -y openjdk-8-jdk
$ wget http://downloads.typesafe.com/scala/2.11.7/scala-2.11.7.tgz
$ tar xzf scala-2.11.7.tgz 
$ mkdir -p /usr/local/share/scala && mv scala-2.11.7 /usr/local/share/scala
$ touch /etc/profile.d/scalahome.sh
$ vim /etc/profile.d/scalahome.sh 

***scalahome.sh***
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
export SCALA_HOME=/usr/local/share/scala/scala-2.11.7
export PATH=$PATH:$SCALA_HOME/bin
***END scalahome.sh***

$ shutdown -r now
$ cd /opt/kafka/
# mkdir /var/log/kafka
# sed -i "s:log.dirs=/tmp/kafka-logs:log.dirs=/var/log/kafka:" config/server.properties
# sed -i "s:#host.name=localhost:host.name=10.0.0.96:" config/server.properties
# sed -i "s|zookeeper.connect=localhost:2181|zookeeper.connect=10.0.0.92:2181,10.0.0.93:2181,10.0.0.94:2181|" config/server.properties
# sed -i "s:#port=9092:port=9092:" config/server.properties
# sed -i "s:#advertised.host.name=<hostname routable by clients>:advertised.host.name=172.22.0.126:" config/server.properties
# sed -i "s|num.network.threads=3|num.network.threads=8|" config/server.properties 
# sed -i "s|log.retention.hours=168|log.retention.hours=1|" config/server.properties
$ bin/kafka-topics.sh --create --zookeeper 10.0.0.92:2181 --replication-factor 1 --partitions 3 --topic test0
$ bin/kafka-topics.sh --list --zookeeper 10.0.0.92:2181
$ bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test0
$ bin/kafka-console-consumer.sh --zookeeper 10.0.0.92:2181 --topic test0 --from-beginning
$ /opt/kafka/bin/kafka-topics.sh --list --zookeeper 10.0.0.92:2181
$ touch /etc/init/kafka-start.conf 
$ vim /etc/init/kafka-start.conf 

***kafka-start.conf***
description "A service to start kafka on startup."
author "bkarels"

start on runlevel [2345]
stop on runlevel [!2345]

respawn

exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties
***END kafka-start.conf***


Init Kafka server config from startup out:

zookeeper.connect=10.0.0.92:2181,10.0.0.93:2181,10.0.0.94:2181

# Timeout in ms for connecting to zookeeper
zookeeper.connection.timeout.ms=6000
root@disposable1:/opt/kafka# bin/kafka-server-start.sh config/server.properties
[2015-12-02 17:57:57,324] INFO KafkaConfig values: 
	advertised.host.name = 172.22.0.126
	metric.reporters = []
	quota.producer.default = 9223372036854775807
	offsets.topic.num.partitions = 50
	log.flush.interval.messages = 9223372036854775807
	auto.create.topics.enable = true
	controller.socket.timeout.ms = 30000
	log.flush.interval.ms = null
	principal.builder.class = class org.apache.kafka.common.security.auth.DefaultPrincipalBuilder
	replica.socket.receive.buffer.bytes = 65536
	min.insync.replicas = 1
	replica.fetch.wait.max.ms = 500
	num.recovery.threads.per.data.dir = 1
	ssl.keystore.type = JKS
	default.replication.factor = 1
	ssl.truststore.password = null
	log.preallocate = false
	sasl.kerberos.principal.to.local.rules = [DEFAULT]
	fetch.purgatory.purge.interval.requests = 1000
	ssl.endpoint.identification.algorithm = null
	replica.socket.timeout.ms = 30000
	message.max.bytes = 1000012
	num.io.threads = 8
	offsets.commit.required.acks = -1
	log.flush.offset.checkpoint.interval.ms = 60000
	delete.topic.enable = false
	quota.window.size.seconds = 1
	ssl.truststore.type = JKS
	offsets.commit.timeout.ms = 5000
	quota.window.num = 11
	zookeeper.connect = 10.0.0.92:2181,10.0.0.93:2181,10.0.0.94:2181
	authorizer.class.name = 
	num.replica.fetchers = 1
	log.retention.ms = null
	log.roll.jitter.hours = 0
	log.cleaner.enable = false
	offsets.load.buffer.size = 5242880
	log.cleaner.delete.retention.ms = 86400000
	ssl.client.auth = none
	controlled.shutdown.max.retries = 3
	queued.max.requests = 500
	offsets.topic.replication.factor = 3
	log.cleaner.threads = 1
	sasl.kerberos.service.name = null
	sasl.kerberos.ticket.renew.jitter = 0.05
	socket.request.max.bytes = 104857600
	ssl.trustmanager.algorithm = PKIX
	zookeeper.session.timeout.ms = 6000
	log.retention.bytes = -1
	sasl.kerberos.min.time.before.relogin = 60000
	zookeeper.set.acl = false
	connections.max.idle.ms = 600000
	offsets.retention.minutes = 1440
	replica.fetch.backoff.ms = 1000
	inter.broker.protocol.version = 0.9.0.X
	log.retention.hours = 1
	num.partitions = 1
	listeners = PLAINTEXT://:9092
	ssl.provider = null
	ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
	log.roll.ms = null
	log.flush.scheduler.interval.ms = 9223372036854775807
	ssl.cipher.suites = null
	log.index.size.max.bytes = 10485760
	ssl.keymanager.algorithm = SunX509
	security.inter.broker.protocol = PLAINTEXT
	replica.fetch.max.bytes = 1048576
	advertised.port = null
	log.cleaner.dedupe.buffer.size = 524288000
	replica.high.watermark.checkpoint.interval.ms = 5000
	log.cleaner.io.buffer.size = 524288
	sasl.kerberos.ticket.renew.window.factor = 0.8
	zookeeper.connection.timeout.ms = 6000
	controlled.shutdown.retry.backoff.ms = 5000
	log.roll.hours = 168
	log.cleanup.policy = delete
	host.name = 10.0.0.96
	log.roll.jitter.ms = null
	max.connections.per.ip = 2147483647
	offsets.topic.segment.bytes = 104857600
	background.threads = 10
	quota.consumer.default = 9223372036854775807
	request.timeout.ms = 30000
	log.index.interval.bytes = 4096
	log.dir = /tmp/kafka-logs
	log.segment.bytes = 1073741824
	log.cleaner.backoff.ms = 15000
	offset.metadata.max.bytes = 4096
	ssl.truststore.location = null
	group.max.session.timeout.ms = 30000
	ssl.keystore.password = null
	zookeeper.sync.time.ms = 2000
	port = 9092
	log.retention.minutes = null
	log.segment.delete.delay.ms = 60000
	log.dirs = /var/log/kafka
	controlled.shutdown.enable = true
	compression.type = producer
	max.connections.per.ip.overrides = 
	sasl.kerberos.kinit.cmd = /usr/bin/kinit
	log.cleaner.io.max.bytes.per.second = 1.7976931348623157E308
	auto.leader.rebalance.enable = true
	leader.imbalance.check.interval.seconds = 300
	log.cleaner.min.cleanable.ratio = 0.5
	replica.lag.time.max.ms = 10000
	num.network.threads = 8
	ssl.key.password = null
	reserved.broker.max.id = 1000
	metrics.num.samples = 2
	socket.send.buffer.bytes = 102400
	ssl.protocol = TLS
	socket.receive.buffer.bytes = 102400
	ssl.keystore.location = null
	replica.fetch.min.bytes = 1
	unclean.leader.election.enable = true
	group.min.session.timeout.ms = 6000
	log.cleaner.io.buffer.load.factor = 0.9
	offsets.retention.check.interval.ms = 600000
	producer.purgatory.purge.interval.requests = 1000
	metrics.sample.window.ms = 30000
	broker.id = 0
	offsets.topic.compression.codec = 0
	log.retention.check.interval.ms = 300000
	advertised.listeners = null
	leader.imbalance.per.broker.percentage = 10
 (kafka.server.KafkaConfig)