Skip to content

Instantly share code, notes, and snippets.

@cjmatta

cjmatta/README.md

Last active August 22, 2019 12:43
Show Gist options
  • Save cjmatta/a716fa26bb1ed22dd7f8d66f2b87d1cd to your computer and use it in GitHub Desktop.
Save cjmatta/a716fa26bb1ed22dd7f8d66f2b87d1cd to your computer and use it in GitHub Desktop.
Download ZIP
Confluent Platform Docker
Raw
README.md

Complete Confluent Platform docker-compose.yml file. Includes an nginx configuration to load-balance between the rest-proxy and schema-registry components.

Running - make sure both docker-compose.yml and nginx_kafka.conf are in the same directory:

$ docker-compose up
Raw
docker-compose.yml
version: '2'
networks:
zk:
driver: bridge
services:
zookeeper-1:
image: confluentinc/cp-zookeeper:latest
environment:
ZOOKEEPER_SERVER_ID: 1
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_INIT_LIMIT: 5
ZOOKEEPER_SYNC_LIMIT: 2
ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: INFO
ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:32888:33888;zookeeper-3:42888:43888
networks:
- default
- zk
ports:
- 22181:2181
- 22888:2888
- 23888:3888
zookeeper-2:
image: confluentinc/cp-zookeeper:latest
environment:
ZOOKEEPER_SERVER_ID: 2
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_INIT_LIMIT: 5
ZOOKEEPER_SYNC_LIMIT: 2
ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: INFO
ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:32888:33888;zookeeper-3:42888:43888
networks:
- default
- zk
ports:
- 32181:2181
- 32888:2888
- 33888:3888
zookeeper-3:
image: confluentinc/cp-zookeeper:latest
environment:
ZOOKEEPER_SERVER_ID: 3
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_INIT_LIMIT: 5
ZOOKEEPER_SYNC_LIMIT: 2
ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: INFO
ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:32888:33888;zookeeper-3:42888:43888
networks:
- default
- zk
ports:
- 42181:2181
- 42888:2888
- 43888:3888
kafka-1:
image: confluentinc/cp-kafka:latest
networks:
- default
- zk
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
ports:
- 19092:9092
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:19092
KAFKA_LOG4J_ROOT_LOGLEVEL: INFO
kafka-2:
image: confluentinc/cp-kafka:latest
networks:
- default
- zk
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
ports:
- 29092:9092
environment:
KAFKA_BROKER_ID: 2
KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:29092
KAFKA_LOG4J_ROOT_LOGLEVEL: INFO
kafka-3:
image: confluentinc/cp-kafka:latest
networks:
- default
- zk
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
ports:
- 39092:9092
environment:
KAFKA_BROKER_ID: 3
KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:39092
KAFKA_LOG4J_ROOT_LOGLEVEL: INFO
control-center:
image: confluentinc/cp-control-center:latest
networks:
- zk
- default
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
- kafka-1
- kafka-2
- kafka-3
environment:
ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181"
BOOTSTRAP_SERVERS: "kafka-1:19092,kafka-2:29092,kafka-3:39092"
CONTROL_CENTER_REPLICATION_FACTOR: 3
MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 6
CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 6
CONTROL_CENTER_STREAM_THREADS: 2
CONTROL_CENTER_LOG4J_ROOT_LOGLEVEL: INFO
ports:
- 9021:9021
schema-registry-1:
image: confluentinc/cp-schema-registry:latest
networks:
- zk
- default
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
- kafka-1
- kafka-2
- kafka-3
environment:
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
SCHEMA_REGISTRY_HOST_NAME: schema-registry-1
SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
ports:
- 18081:8081
schema-registry-2:
image: confluentinc/cp-schema-registry:latest
networks:
- zk
- default
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
- kafka-1
- kafka-2
- kafka-3
environment:
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
SCHEMA_REGISTRY_HOST_NAME: schema-registry-2
SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
ports:
- 28081:8081
schema-registry-3:
image: confluentinc/cp-schema-registry:latest
networks:
- zk
- default
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
- kafka-1
- kafka-2
- kafka-3
environment:
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
SCHEMA_REGISTRY_HOST_NAME: schema-registry-3
SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
ports:
- 38081:8081
restproxy-1:
image: confluentinc/cp-kafka-rest:latest
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
- schema-registry-1
- schema-registry-2
- schema-registry-3
ports:
- 18082:8082
environment:
KAFKA_REST_HOST_NAME: restproxy-1
KAFKA_REST_ID_CONFIG: 1
KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
KAFKA_REST_LOG4J_ROOT_LOGLEVEL: INFO
restproxy-2:
image: confluentinc/cp-kafka-rest:latest
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
- schema-registry-1
- schema-registry-2
- schema-registry-3
ports:
- 28082:8082
environment:
KAFKA_REST_HOST_NAME: restproxy-2
KAFKA_REST_ID_CONFIG: 2
KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
KAFKA_REST_LOG4J_ROOT_LOGLEVEL: INFO
restproxy-3:
image: confluentinc/cp-kafka-rest:latest
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
- schema-registry-1
- schema-registry-2
- schema-registry-3
ports:
- 38082:8082
environment:
KAFKA_REST_HOST_NAME: restproxy-3
KAFKA_REST_ID_CONFIG: 3
KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
KAFKA_REST_LOG4J_ROOT_LOGLEVEL: INFO
connect-1:
image: confluentinc/cp-kafka-connect:latest
restart: on-failure:3
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
- kafka-1
- kafka-2
- kafka-3
# include Kafka Connectors
# these volumes should have the files from the `target/kafka-connect-twitter-1.0-SNAPSHOT-pacage/share/java` dir
# after running mvn package
# volumes:
# - ./connectors/kafka-connect-twitter:/usr/share/java/kafka-connect-twitter
# - ./connectors/kafka-connect-spooldir:/usr/share/java/kafka-connect-spooldir
environment:
CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: "default"
CONNECT_CONFIG_STORAGE_TOPIC: "default.config"
CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets"
CONNECT_STATUS_STORAGE_TOPIC: "default.status"
CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_REST_ADVERTISED_HOST_NAME: "connect-1"
CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
connect-2:
image: confluentinc/cp-kafka-connect:latest
restart: on-failure:3
links:
- zookeeper-1
- zookeeper-2
- zookeeper-3
- kafka-1
- kafka-2
- kafka-3
# include Kafka Connectors
# these volumes should have the files from the `target/kafka-connect-twitter-1.0-SNAPSHOT-pacage/share/java` dir
# after running mvn package
# volumes:
# - ./connectors/kafka-connect-twitter:/usr/share/java/kafka-connect-twitter
# - ./connectors/kafka-connect-spooldir:/usr/share/java/kafka-connect-spooldir
environment:
CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: "default"
CONNECT_CONFIG_STORAGE_TOPIC: "default.config"
CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets"
CONNECT_STATUS_STORAGE_TOPIC: "default.status"
CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_REST_ADVERTISED_HOST_NAME: "connect-2"
CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
nginx:
image: nginx:latest
ports:
- "8080:8080"
volumes:
- ./nginx_kafka.conf:/etc/nginx/conf.d/kafka_proxy.template
links:
- restproxy-1
- restproxy-2
- restproxy-3
- schema-registry-1
- schema-registry-2
- schema-registry-3
- control-center
environment:
- NGINX_HOST=confluent-platform
- NGINX_PORT=8080
command: /bin/bash -c "envsubst < /etc/nginx/conf.d/kafka_proxy.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"
Raw
nginx_kafka.conf
upstream restproxy {
ip_hash;
server restproxy-1:8082;
server restproxy-2:8082;
server restproxy-3:8082;
}
upstream schemaregistry {
ip_hash;
server schema-registry-1:8081;
server schema-registry-2:8081;
server schema-registry-3:8081;
}
server {
server_name rest-proxy;
listen 8082;
location / {
proxy_pass http://restproxy;
}
}
server {
server_name schema-registry;
listen 8081;
location / {
proxy_pass http://schemaregistry;
}
}
server {
server_name restproxy-1;
location / {
proxy_pass http://restproxy-1:8082;
}
}
server {
server_name restproxy-2;
location / {
proxy_pass http://restproxy-2:8082;
}
}
server {
server_name restproxy-3;
location / {
proxy_pass http://restproxy-3:8082;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment