Skip to content

Instantly share code, notes, and snippets.

@aulisius

aulisius/Dockerfile

Last active April 23, 2020 07:41
Show Gist options
  • Save aulisius/9593c469a97cac72aa23f13e3e9963a2 to your computer and use it in GitHub Desktop.
Save aulisius/9593c469a97cac72aa23f13e3e9963a2 to your computer and use it in GitHub Desktop.
Download ZIP
Setup Confluent Platform Kafka on OpenShift/k8s
Raw
confluent-kafka.json
{
"kind": "Template",
"apiVersion": "template.openshift.io/v1",
"metadata": { "name": "lebron-confluent-kafka" },
"parameters": [],
"objects": [
{
"kind": "DeploymentConfig",
"apiVersion": "apps.openshift.io/v1",
"metadata": {
"name": "lebron-confluent-kafka",
"labels": { "app": "lebron-confluent-kafka" }
},
"spec": {
"triggers": [{ "type": "ConfigChange" }],
"replicas": 1,
"test": false,
"selector": {
"deploymentconfig": "lebron-confluent-kafka"
},
"template": {
"metadata": {
"labels": {
"deploymentconfig": "lebron-confluent-kafka"
}
},
"spec": {
"hostname": "lebron-confluent-kafka",
"containers": [
{
"imagePullPolicy": "Always",
"name": "broker",
"image": "artifactory.webhostbox.net:9011/confluentinc/cp-server:latest",
"ports": [{ "containerPort": 9092 }],
"env": [
{ "name": "CONFLUENT_METRICS_ENABLE", "value": "true" },
{
"name": "CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS",
"value": "localhost:29092"
},
{
"name": "CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS",
"value": "1"
},
{
"name": "CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT",
"value": "localhost:2181"
},
{
"name": "CONFLUENT_SUPPORT_CUSTOMER_ID",
"value": "anonymous"
},
{
"name": "KAFKA_ADVERTISED_LISTENERS",
"value": "PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092"
},
{ "name": "KAFKA_BROKER_ID", "value": "1" },
{
"name": "KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR",
"value": "1"
},
{
"name": "KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS",
"value": "0"
},
{
"name": "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP",
"value": "PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT"
},
{
"name": "KAFKA_METRIC_REPORTERS",
"value": "io.confluent.metrics.reporter.ConfluentMetricsReporter"
},
{
"name": "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR",
"value": "1"
},
{
"name": "KAFKA_ZOOKEEPER_CONNECT",
"value": "localhost:2181"
},
{
"name": "KAFKA_HEAP_OPTS",
"value": "-Xms16G -Xmx16G"
}
],
"volumeMounts": [
{ "mountPath": "/var/lib/kafka/data", "name": "kafka" },
{ "mountPath": "/etc/kafka/secrets", "name": "kafka" }
]
},
{
"imagePullPolicy": "Always",
"name": "zookeeper",
"image": "artifactory.webhostbox.net:9011/confluentinc/cp-zookeeper:latest",
"ports": [{ "containerPort": 2181 }],
"env": [
{ "name": "ZOOKEEPER_CLIENT_PORT", "value": "2181" },
{ "name": "ZOOKEEPER_TICK_TIME", "value": "2000" }
],
"volumeMounts": [
{ "mountPath": "/var/lib/zookeeper/data", "name": "kafka" },
{ "mountPath": "/etc/zookeeper/secrets", "name": "kafka" },
{ "mountPath": "/var/lib/zookeeper/log", "name": "kafka" }
]
},
{
"imagePullPolicy": "Always",
"name": "schema-registry",
"image": "artifactory.webhostbox.net:9011/confluentinc/cp-schema-registry:latest",
"ports": [{ "containerPort": 8081 }],
"env": [
{
"name": "SCHEMA_REGISTRY_HOST_NAME",
"value": "schema-registry"
},
{
"name": "SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL",
"value": "localhost:2181"
}
],
"volumeMounts": [
{
"mountPath": "/etc/schema-registry/secrets",
"name": "kafka"
}
],
"resources": {}
},
{
"imagePullPolicy": "Always",
"name": "rest-proxy",
"image": "artifactory.webhostbox.net:9011/confluentinc/cp-kafka-rest:latest",
"ports": [{ "containerPort": 8082 }],
"env": [
{
"name": "KAFKA_REST_BOOTSTRAP_SERVERS",
"value": "localhost:29092"
},
{ "name": "KAFKA_REST_HOST_NAME", "value": "rest-proxy" },
{
"name": "KAFKA_REST_LISTENERS",
"value": "http://0.0.0.0:8082"
},
{
"name": "KAFKA_REST_SCHEMA_REGISTRY_URL",
"value": "http://localhost:8081"
}
],
"resources": {}
},
{
"imagePullPolicy": "Always",
"name": "connect",
"image": "artifactory.webhostbox.net:9011/confluentinc/cp-server-connect:latest",
"ports": [{ "containerPort": 8083 }],
"env": [
{
"name": "CLASSPATH",
"value": "/usr/share/java/monitoring-interceptors/monitoring-interceptors-5.4.1.jar"
},
{
"name": "CONNECT_BOOTSTRAP_SERVERS",
"value": "localhost:29092"
},
{
"name": "CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR",
"value": "1"
},
{
"name": "CONNECT_CONFIG_STORAGE_TOPIC",
"value": "docker-connect-configs"
},
{
"name": "CONNECT_CONSUMER_INTERCEPTOR_CLASSES",
"value": "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
},
{
"name": "CONNECT_GROUP_ID",
"value": "compose-connect-group"
},
{
"name": "CONNECT_INTERNAL_KEY_CONVERTER",
"value": "org.apache.kafka.connect.json.JsonConverter"
},
{
"name": "CONNECT_INTERNAL_VALUE_CONVERTER",
"value": "org.apache.kafka.connect.json.JsonConverter"
},
{
"name": "CONNECT_KEY_CONVERTER",
"value": "org.apache.kafka.connect.storage.StringConverter"
},
{
"name": "CONNECT_LOG4J_LOGGERS",
"value": "org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR"
},
{
"name": "CONNECT_OFFSET_FLUSH_INTERVAL_MS",
"value": "10000"
},
{
"name": "CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR",
"value": "1"
},
{
"name": "CONNECT_OFFSET_STORAGE_TOPIC",
"value": "docker-connect-offsets"
},
{
"name": "CONNECT_PLUGIN_PATH",
"value": "/usr/share/java,/usr/share/confluent-hub-components"
},
{
"name": "CONNECT_PRODUCER_INTERCEPTOR_CLASSES",
"value": "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
},
{
"name": "CONNECT_REST_ADVERTISED_HOST_NAME",
"value": "connect"
},
{ "name": "CONNECT_REST_PORT", "value": "8083" },
{
"name": "CONNECT_STATUS_STORAGE_REPLICATION_FACTOR",
"value": "1"
},
{
"name": "CONNECT_STATUS_STORAGE_TOPIC",
"value": "docker-connect-status"
},
{
"name": "CONNECT_VALUE_CONVERTER",
"value": "io.confluent.connect.avro.AvroConverter"
},
{
"name": "CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL",
"value": "http://localhost:8081"
},
{
"name": "CONNECT_ZOOKEEPER_CONNECT",
"value": "localhost:2181"
}
]
}
],
"volumes": [{ "name": "kafka", "emptyDir": {} }],
"restartPolicy": "Always",
"terminationGracePeriodSeconds": 30,
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"schedulerName": "default-scheduler"
}
}
}
},
{
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": "lebron-confluent-kafka"
},
"spec": {
"ports": [
{ "name": "9092", "port": 9092, "targetPort": 9092 },
{ "name": "2181", "port": 2181, "targetPort": 2181 },
{ "name": "8081", "port": 8081, "targetPort": 8081 },
{ "name": "8082", "port": 8082, "targetPort": 8082 },
{ "name": "8083", "port": 8083, "targetPort": 8083 },
{ "name": "8088", "port": 8088, "targetPort": 8088 }
],
"selector": { "deploymentconfig": "lebron-confluent-kafka" },
"type": "ClusterIP",
"sessionAffinity": "None"
},
"status": { "loadBalancer": {} }
}
]
}
Raw
Dockerfile
FROM confluentinc/cp-server-connect
RUN confluent-hub install debezium/debezium-connector-postgresql:0.10.0 --no-prompt
RUN confluent-hub install debezium/debezium-connector-mysql:1.1.0 --no-prompt
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment