Skip to content

Instantly share code, notes, and snippets.

@dasbh

dasbh/config.yml

Last active November 15, 2019 23:45
Show Gist options
  • Save dasbh/649f2cf92fe97ea4ae18981ecb68f93b to your computer and use it in GitHub Desktop.
Save dasbh/649f2cf92fe97ea4ae18981ecb68f93b to your computer and use it in GitHub Desktop.
Download ZIP
Confluent Platform Kube manifest
Raw
config.yml
apiVersion: v1
name: app
namespace: kafka
charts:
- confluentinc/cp-helm-charts:
values:
cp-zookeeper:
enabled: true
servers: 3
image: confluentinc/cp-zookeeper
imageTag: 5.3.1-1
## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace.
## https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod
imagePullSecrets:
# - name: "regcred"
heapOptions: "-Xms512M -Xmx512M"
persistence:
enabled: true
## The size of the PersistentVolume to allocate to each Zookeeper Pod in the StatefulSet. For
## production servers this number should likely be much larger.
##
## Size for Data dir, where ZooKeeper will store the in-memory database snapshots.
dataDirSize: 10Gi
# dataDirStorageClass: ""
## Size for data log dir, which is a dedicated log device to be used, and helps avoid competition between logging and snaphots.
dataLogDirSize: 10Gi
# dataLogDirStorageClass: ""
resources:
## If you do want to specify resources, uncomment the following lines, adjust them as necessary,
## and remove the curly braces after 'resources:'
limits:
cpu: 100m
memory: 256Mi
requests:
cpu: 100m
memory: 256Mi
cp-kafka:
enabled: true
brokers: 3
image: confluentinc/cp-enterprise-kafka
imageTag: 5.3.1-1
## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace.
## https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod
imagePullSecrets:
# - name: "regcred"
heapOptions: "-Xms512M -Xmx512M"
persistence:
enabled: true
# storageClass: ""
size: 5Gi
disksPerBroker: 1
resources: {}
## If you do want to specify resources, uncomment the following lines, adjust them as necessary,
## and remove the curly braces after 'resources:'
# limits:
# cpu: 500m
# memory: 512Mi
# requests:
# cpu: 500m
# memory: 512Mi
cp-schema-registry:
enabled: true
image: confluentinc/cp-schema-registry
imageTag: 5.3.1-1
## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace.
## https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod
imagePullSecrets:
# - name: "regcred"
heapOptions: "-Xms512M -Xmx512M"
resources: {}
## If you do want to specify resources, uncomment the following lines, adjust them as necessary,
## and remove the curly braces after 'resources:'
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
## ------------------------------------------------------
## REST Proxy
## ------------------------------------------------------
cp-kafka-rest:
enabled: true
image: confluentinc/cp-kafka-rest
imageTag: 5.3.1-1
## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace.
## https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod
imagePullSecrets:
# - name: "regcred"
heapOptions: "-Xms512M -Xmx512M"
resources: {}
## If you do want to specify resources, uncomment the following lines, adjust them as necessary,
## and remove the curly braces after 'resources:'
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
## ------------------------------------------------------
## Kafka Connect
## ------------------------------------------------------
cp-kafka-connect:
enabled: true
image: confluentinc/cp-kafka-connect
imageTag: 5.3.1-1
## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace.
## https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod
imagePullSecrets:
# - name: "regcred"
heapOptions: "-Xms512M -Xmx512M"
resources: {}
## If you do want to specify resources, uncomment the following lines, adjust them as necessary,
## and remove the curly braces after 'resources:'
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
Raw
deployment.yml
---
apiVersion: v1
kind: Pod
metadata:
name: "my-dep-apps-canary"
annotations:
"helm.sh/hook": test-success
"helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded"
spec:
containers:
- name: my-dep-apps-canary
image: "confluentinc/cp-enterprise-kafka:5.3.1-1"
imagePullPolicy: "IfNotPresent"
command:
- sh
- -c
- |
# Delete the topic if it exists
kafka-topics --zookeeper my-dep-apps-cp-zookeeper-headless:2181 --topic my-dep-apps-cp-kafka-canary-topic --delete --if-exists
# Create the topic
kafka-topics --zookeeper my-dep-apps-cp-zookeeper-headless:2181 --topic my-dep-apps-cp-kafka-canary-topic --create --partitions 1 --replication-factor 1 --if-not-exists && \
# Create a message
MESSAGE="`date -u`" && \
# Produce a test message to the topic
echo "$MESSAGE" | kafka-console-producer --broker-list my-dep-apps-cp-kafka:9092 --topic my-dep-apps-cp-kafka-canary-topic && \
# Consume a test message from the topic
kafka-console-consumer --bootstrap-server my-dep-apps-cp-kafka-headless:9092 --topic my-dep-apps-cp-kafka-canary-topic --from-beginning --timeout-ms 2000 | grep "$MESSAGE"
restartPolicy: Never
---
apiVersion: v1
kind: ConfigMap
metadata:
name: my-dep-apps-cp-schema-registry-jmx-configmap
labels:
app: cp-schema-registry
chart: cp-schema-registry-0.1.0
release: my-dep-apps
heritage: Tiller
data:
jmx-schema-registry-prometheus.yml: |+
jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
lowercaseOutputName: true
lowercaseOutputLabelNames: true
ssl: false
rules:
- pattern : 'kafka.schema.registry<type=jetty-metrics>([^:]+):'
name: "cp_kafka_schema_registry_jetty_metrics_$1"
- pattern : 'kafka.schema.registry<type=master-slave-role>([^:]+):'
name: "cp_kafka_schema_registry_master_slave_role"
- pattern : 'kafka.schema.registry<type=jersey-metrics>([^:]+):'
name: "cp_kafka_schema_registry_jersey_metrics_$1"
---
apiVersion: v1
kind: Service
metadata:
name: my-dep-apps-cp-ksql-server
labels:
app: cp-ksql-server
chart: cp-ksql-server-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
ports:
- name: ksql-server
port: 8088
selector:
app: cp-ksql-server
release: my-dep-apps
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: my-dep-apps-cp-zookeeper-pdb
labels:
app: cp-zookeeper
chart: cp-zookeeper-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
selector:
matchLabels:
app: cp-zookeeper
release: my-dep-apps
maxUnavailable: 1
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: my-dep-apps-cp-kafka
labels:
app: cp-kafka
chart: cp-kafka-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
serviceName: my-dep-apps-cp-kafka-headless
podManagementPolicy: OrderedReady
replicas: 3
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: cp-kafka
release: my-dep-apps
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "5556"
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
podAffinityTerm:
labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- cp-kafka
- key: "release"
operator: In
values:
- my-dep-apps
topologyKey: "kubernetes.io/hostname"
containers:
- name: prometheus-jmx-exporter
image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
command:
- java
- -XX:+UnlockExperimentalVMOptions
- -XX:+UseCGroupMemoryLimitForHeap
- -XX:MaxRAMFraction=1
- -XshowSettings:vm
- -jar
- jmx_prometheus_httpserver.jar
- "5556"
- /etc/jmx-kafka/jmx-kafka-prometheus.yml
ports:
- containerPort: 5556
resources:
null
volumeMounts:
- name: jmx-config
mountPath: /etc/jmx-kafka
- name: cp-kafka-broker
image: "confluentinc/cp-enterprise-kafka:5.3.1-1"
imagePullPolicy: "IfNotPresent"
ports:
- containerPort: 9092
name: kafka
- containerPort: 5555
name: jmx
resources:
{}
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: KAFKA_HEAP_OPTS
value: -Xms512M -Xmx512M
- name: KAFKA_ZOOKEEPER_CONNECT
value: "my-dep-apps-cp-zookeeper-headless:2181"
- name: KAFKA_LOG_DIRS
value: "/opt/kafka/data-0/logs"
- name: "KAFKA_ADVERTISED_LISTENERS"
value: "EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID}))"
- name: "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP"
value: "PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT"
- name: "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR"
value: "3"
- name: KAFKA_JMX_PORT
value: "5555"
# This is required because the Downward API does not yet support identification of
# pod numbering in statefulsets. Thus, we are required to specify a command which
# allows us to extract the pod ID for usage as the Kafka Broker ID.
# See: https://github.com/kubernetes/kubernetes/issues/31218
command:
- sh
- -exc
- |
export KAFKA_BROKER_ID=${HOSTNAME##*-} && \
export KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${POD_IP}:9092,EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID})) && \
exec /etc/confluent/docker/run
volumeMounts:
- name: datadir-0
mountPath: /opt/kafka/data-0
volumes:
- name: jmx-config
configMap:
name: my-dep-apps-cp-kafka-jmx-configmap
volumeClaimTemplates:
- metadata:
name: datadir-0
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: "5Gi"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: my-dep-apps-cp-kafka-jmx-configmap
labels:
app: cp-kafka
chart: cp-kafka-0.1.0
release: my-dep-apps
heritage: Tiller
data:
jmx-kafka-prometheus.yml: |+
jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
lowercaseOutputName: true
lowercaseOutputLabelNames: true
ssl: false
rules:
- pattern : kafka.server<type=ReplicaManager, name=(.+)><>(Value|OneMinuteRate)
name: "cp_kafka_server_replicamanager_$1"
- pattern : kafka.controller<type=KafkaController, name=(.+)><>Value
name: "cp_kafka_controller_kafkacontroller_$1"
- pattern : kafka.server<type=BrokerTopicMetrics, name=(.+)><>OneMinuteRate
name: "cp_kafka_server_brokertopicmetrics_$1"
- pattern : kafka.network<type=RequestMetrics, name=RequestsPerSec, request=(.+)><>OneMinuteRate
name: "cp_kafka_network_requestmetrics_requestspersec_$1"
- pattern : kafka.network<type=SocketServer, name=NetworkProcessorAvgIdlePercent><>Value
name: "cp_kafka_network_socketserver_networkprocessoravgidlepercent"
- pattern : kafka.server<type=ReplicaFetcherManager, name=MaxLag, clientId=(.+)><>Value
name: "cp_kafka_server_replicafetchermanager_maxlag_$1"
- pattern : kafka.server<type=KafkaRequestHandlerPool, name=RequestHandlerAvgIdlePercent><>OneMinuteRate
name: "cp_kafka_kafkarequesthandlerpool_requesthandleravgidlepercent"
- pattern : kafka.controller<type=ControllerStats, name=(.+)><>OneMinuteRate
name: "cp_kafka_controller_controllerstats_$1"
- pattern : kafka.server<type=SessionExpireListener, name=(.+)><>OneMinuteRate
name: "cp_kafka_server_sessionexpirelistener_$1"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: my-dep-apps-cp-kafka-rest-jmx-configmap
labels:
app: cp-kafka-rest
chart: cp-kafka-rest-0.1.0
release: my-dep-apps
heritage: Tiller
data:
jmx-kafka-rest-prometheus.yml: |+
jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
lowercaseOutputName: true
lowercaseOutputLabelNames: true
ssl: false
rules:
- pattern : 'kafka.rest<type=jetty-metrics>([^:]+):'
name: "cp_kafka_rest_jetty_metrics_$1"
- pattern : 'kafka.rest<type=jersey-metrics>([^:]+):'
name: "cp_kafka_rest_jersey_metrics_$1"
---
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: my-dep-apps-cp-kafka-connect
labels:
app: cp-kafka-connect
chart: cp-kafka-connect-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
replicas: 1
selector:
matchLabels:
app: cp-kafka-connect
release: my-dep-apps
template:
metadata:
labels:
app: cp-kafka-connect
release: my-dep-apps
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "5556"
spec:
containers:
- name: prometheus-jmx-exporter
image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
command:
- java
- -XX:+UnlockExperimentalVMOptions
- -XX:+UseCGroupMemoryLimitForHeap
- -XX:MaxRAMFraction=1
- -XshowSettings:vm
- -jar
- jmx_prometheus_httpserver.jar
- "5556"
- /etc/jmx-kafka-connect/jmx-kafka-connect-prometheus.yml
ports:
- containerPort: 5556
resources:
null
volumeMounts:
- name: jmx-config
mountPath: /etc/jmx-kafka-connect
- name: cp-kafka-connect-server
image: "confluentinc/cp-kafka-connect:5.3.1-1"
imagePullPolicy: "IfNotPresent"
ports:
- name: kafka-connect
containerPort: 8083
protocol: TCP
- containerPort: 5555
name: jmx
resources:
{}
env:
- name: CONNECT_REST_ADVERTISED_HOST_NAME
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: CONNECT_BOOTSTRAP_SERVERS
value: PLAINTEXT://my-dep-apps-cp-kafka-headless:9092
- name: CONNECT_GROUP_ID
value: my-dep-apps
- name: CONNECT_CONFIG_STORAGE_TOPIC
value: my-dep-apps-cp-kafka-connect-config
- name: CONNECT_OFFSET_STORAGE_TOPIC
value: my-dep-apps-cp-kafka-connect-offset
- name: CONNECT_STATUS_STORAGE_TOPIC
value: my-dep-apps-cp-kafka-connect-status
- name: CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL
value: http://my-dep-apps-cp-schema-registry:8081
- name: CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL
value: http://my-dep-apps-cp-schema-registry:8081
- name: "CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR"
value: "3"
- name: "CONNECT_INTERNAL_KEY_CONVERTER"
value: "org.apache.kafka.connect.json.JsonConverter"
- name: "CONNECT_INTERNAL_VALUE_CONVERTER"
value: "org.apache.kafka.connect.json.JsonConverter"
- name: "CONNECT_KEY_CONVERTER"
value: "io.confluent.connect.avro.AvroConverter"
- name: "CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE"
value: "false"
- name: "CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR"
value: "3"
- name: "CONNECT_PLUGIN_PATH"
value: "/usr/share/java"
- name: "CONNECT_STATUS_STORAGE_REPLICATION_FACTOR"
value: "3"
- name: "CONNECT_VALUE_CONVERTER"
value: "io.confluent.connect.avro.AvroConverter"
- name: "CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE"
value: "false"
- name: KAFKA_JMX_PORT
value: "5555"
volumes:
- name: jmx-config
configMap:
name: my-dep-apps-cp-kafka-connect-jmx-configmap
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: my-dep-apps-cp-zookeeper
labels:
app: cp-zookeeper
chart: cp-zookeeper-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
serviceName: my-dep-apps-cp-zookeeper-headless
podManagementPolicy: OrderedReady
replicas: 3
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: cp-zookeeper
release: my-dep-apps
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "5556"
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
podAffinityTerm:
labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- cp-zookeeper
- key: "release"
operator: In
values:
- my-dep-apps
topologyKey: "kubernetes.io/hostname"
containers:
- name: prometheus-jmx-exporter
image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
command:
- java
- -XX:+UnlockExperimentalVMOptions
- -XX:+UseCGroupMemoryLimitForHeap
- -XX:MaxRAMFraction=1
- -XshowSettings:vm
- -jar
- jmx_prometheus_httpserver.jar
- "5556"
- /etc/jmx-zookeeper/jmx-zookeeper-prometheus.yml
ports:
- containerPort: 5556
resources:
null
volumeMounts:
- name: jmx-config
mountPath: /etc/jmx-zookeeper
- name: cp-zookeeper-server
image: "confluentinc/cp-zookeeper:5.3.1-1"
imagePullPolicy: "IfNotPresent"
ports:
- containerPort: 2181
name: client
- containerPort: 2888
name: server
- containerPort: 3888
name: leader-election
- containerPort: 5555
name: jmx
resources:
limits:
cpu: 100m
memory: 256Mi
requests:
cpu: 100m
memory: 256Mi
env:
- name : KAFKA_HEAP_OPTS
value: "-Xms512M -Xmx512M"
- name : KAFKA_JMX_PORT
value: "5555"
- name : ZOOKEEPER_TICK_TIME
value: "2000"
- name : ZOOKEEPER_SYNC_LIMIT
value: "5"
- name : ZOOKEEPER_INIT_LIMIT
value: "10"
- name : ZOOKEEPER_MAX_CLIENT_CNXNS
value: "60"
- name : ZOOKEEPER_AUTOPURGE_SNAP_RETAIN_COUNT
value: "3"
- name : ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL
value: "24"
- name: ZOOKEEPER_CLIENT_PORT
value: "2181"
- name : ZOOKEEPER_SERVERS
value: "my-dep-apps-cp-zookeeper-0.my-dep-apps-cp-zookeeper-headless.kafka:2888:3888;my-dep-apps-cp-zookeeper-1.my-dep-apps-cp-zookeeper-headless.kafka:2888:3888;my-dep-apps-cp-zookeeper-2.my-dep-apps-cp-zookeeper-headless.kafka:2888:3888"
# ZOOKEEPER_SERVER_ID is required just to pass cp-zookeeper ensure script for env check,
# the value(metadata.mame) is not used and will be overwritten in command part
- name: ZOOKEEPER_SERVER_ID
valueFrom:
fieldRef:
fieldPath: metadata.name
command:
- "bash"
- "-c"
- "ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-}+1)) && /etc/confluent/docker/run"
volumeMounts:
- name: datadir
mountPath: /var/lib/zookeeper/data
- name: datalogdir
mountPath: /var/lib/zookeeper/log
volumes:
- name: jmx-config
configMap:
name: my-dep-apps-cp-zookeeper-jmx-configmap
volumeClaimTemplates:
- metadata:
name: datadir
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: "10Gi"
- metadata:
name: datalogdir
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: "10Gi"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: my-dep-apps-cp-ksql-server-ksql-queries-configmap
labels:
app: cp-ksql-server
chart: cp-ksql-server-0.1.0
release: my-dep-apps
heritage: Tiller
data:
queries.sql: |-
-- From http://docs.confluent.io/current/ksql/docs/tutorials/basics-docker.html#create-a-stream-and-table
-- Create a stream pageviews_original from the Kafka topic pageviews, specifying the value_format of DELIMITED
CREATE STREAM pageviews_original (viewtime bigint, userid varchar, pageid varchar) WITH (kafka_topic='pageviews', value_format='DELIMITED');
-- Create a table users_original from the Kafka topic users, specifying the value_format of JSON
CREATE TABLE users_original (registertime BIGINT, gender VARCHAR, regionid VARCHAR, userid VARCHAR) WITH (kafka_topic='users', value_format='JSON', key = 'userid');
-- Create a persistent query by using the CREATE STREAM keywords to precede the SELECT statement
CREATE STREAM pageviews_enriched AS SELECT users_original.userid AS userid, pageid, regionid, gender FROM pageviews_original LEFT JOIN users_original ON pageviews_original.userid = users_original.userid;
-- Create a new persistent query where a condition limits the streams content, using WHERE
CREATE STREAM pageviews_female AS SELECT * FROM pageviews_enriched WHERE gender = 'FEMALE';
-- Create a new persistent query where another condition is met, using LIKE
CREATE STREAM pageviews_female_like_89 WITH (kafka_topic='pageviews_enriched_r8_r9') AS SELECT * FROM pageviews_female WHERE regionid LIKE '%_8' OR regionid LIKE '%_9';
-- Create a new persistent query that counts the pageviews for each region and gender combination in a tumbling window of 30 seconds when the count is greater than one
CREATE TABLE pageviews_regions WITH (VALUE_FORMAT='avro') AS SELECT gender, regionid , COUNT(*) AS numusers FROM pageviews_enriched WINDOW TUMBLING (size 30 second) GROUP BY gender, regionid HAVING COUNT(*) > 1;
---
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: my-dep-apps-cp-ksql-server
labels:
app: cp-ksql-server
chart: cp-ksql-server-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
replicas: 1
selector:
matchLabels:
app: cp-ksql-server
release: my-dep-apps
template:
metadata:
labels:
app: cp-ksql-server
release: my-dep-apps
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "5556"
spec:
containers:
- name: prometheus-jmx-exporter
image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
command:
- java
- -XX:+UnlockExperimentalVMOptions
- -XX:+UseCGroupMemoryLimitForHeap
- -XX:MaxRAMFraction=1
- -XshowSettings:vm
- -jar
- jmx_prometheus_httpserver.jar
- "5556"
- /etc/jmx-ksql-server/jmx-ksql-server-prometheus.yml
ports:
- containerPort: 5556
resources:
null
volumeMounts:
- name: jmx-config
mountPath: /etc/jmx-ksql-server
- name: cp-ksql-server
image: "confluentinc/cp-ksql-server:5.0.1"
imagePullPolicy: "IfNotPresent"
ports:
- name: server
containerPort: 8088
protocol: TCP
- containerPort: 5555
name: jmx
resources:
{}
volumeMounts:
env:
- name: KSQL_BOOTSTRAP_SERVERS
value: PLAINTEXT://my-dep-apps-cp-kafka-headless:9092
- name: KSQL_KSQL_SERVICE_ID
value: my-dep-apps
- name: KSQL_KSQL_SCHEMA_REGISTRY_URL
value: http://my-dep-apps-cp-schema-registry:8081
- name: KSQL_LISTENERS
value: http://0.0.0.0:8088
- name: KSQL_JMX_PORT
value: "5555"
volumes:
- name: jmx-config
configMap:
name: my-dep-apps-cp-ksql-server-jmx-configmap
---
apiVersion: v1
kind: Service
metadata:
name: my-dep-apps-cp-kafka-rest
labels:
app: cp-kafka-rest
chart: cp-kafka-rest-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
ports:
- name: rest-proxy
port: 8082
selector:
app: cp-kafka-rest
release: my-dep-apps
---
apiVersion: v1
kind: Service
metadata:
name: my-dep-apps-cp-kafka-headless
labels:
app: cp-kafka
chart: cp-kafka-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
ports:
- port: 9092
name: broker
clusterIP: None
selector:
app: cp-kafka
release: my-dep-apps
---
apiVersion: v1
kind: Service
metadata:
name: my-dep-apps-cp-kafka-connect
labels:
app: cp-kafka-connect
chart: cp-kafka-connect-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
ports:
- name: kafka-connect
port: 8083
selector:
app: cp-kafka-connect
release: my-dep-apps
---
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: my-dep-apps-cp-kafka-rest
labels:
app: cp-kafka-rest
chart: cp-kafka-rest-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
replicas: 1
selector:
matchLabels:
app: cp-kafka-rest
release: my-dep-apps
template:
metadata:
labels:
app: cp-kafka-rest
release: my-dep-apps
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "5556"
spec:
containers:
- name: prometheus-jmx-exporter
image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
command:
- java
- -XX:+UnlockExperimentalVMOptions
- -XX:+UseCGroupMemoryLimitForHeap
- -XX:MaxRAMFraction=1
- -XshowSettings:vm
- -jar
- jmx_prometheus_httpserver.jar
- "5556"
- /etc/jmx-kafka-rest/jmx-kafka-rest-prometheus.yml
ports:
- containerPort: 5556
resources:
null
volumeMounts:
- name: jmx-config
mountPath: /etc/jmx-kafka-rest
- name: cp-kafka-rest-server
image: "confluentinc/cp-kafka-rest:5.3.1-1"
imagePullPolicy: "IfNotPresent"
ports:
- name: rest-proxy
containerPort: 8082
protocol: TCP
- containerPort: 5555
name: jmx
resources:
{}
env:
- name: KAFKA_REST_HOST_NAME
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: KAFKA_REST_ZOOKEEPER_CONNECT
value: my-dep-apps-cp-zookeeper-headless:2181
- name: KAFKA_REST_SCHEMA_REGISTRY_URL
value: http://my-dep-apps-cp-schema-registry:8081
- name: KAFKA_REST_JMX_PORT
value: "5555"
volumes:
- name: jmx-config
configMap:
name: my-dep-apps-cp-kafka-rest-jmx-configmap
---
apiVersion: v1
kind: Service
metadata:
name: my-dep-apps-cp-zookeeper
labels:
app: cp-zookeeper
chart: cp-zookeeper-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
type:
ports:
- port: 2181
name: client
selector:
app: cp-zookeeper
release: my-dep-apps
---
apiVersion: v1
kind: Service
metadata:
name: my-dep-apps-cp-schema-registry
labels:
app: cp-schema-registry
chart: cp-schema-registry-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
ports:
- name: schema-registry
port: 8081
selector:
app: cp-schema-registry
release: my-dep-apps
---
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: my-dep-apps-cp-schema-registry
labels:
app: cp-schema-registry
chart: cp-schema-registry-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
replicas: 1
selector:
matchLabels:
app: cp-schema-registry
release: my-dep-apps
template:
metadata:
labels:
app: cp-schema-registry
release: my-dep-apps
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "5556"
spec:
containers:
- name: prometheus-jmx-exporter
image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
command:
- java
- -XX:+UnlockExperimentalVMOptions
- -XX:+UseCGroupMemoryLimitForHeap
- -XX:MaxRAMFraction=1
- -XshowSettings:vm
- -jar
- jmx_prometheus_httpserver.jar
- "5556"
- /etc/jmx-schema-registry/jmx-schema-registry-prometheus.yml
ports:
- containerPort: 5556
resources:
null
volumeMounts:
- name: jmx-config
mountPath: /etc/jmx-schema-registry
- name: cp-schema-registry-server
image: "confluentinc/cp-schema-registry:5.3.1-1"
imagePullPolicy: "IfNotPresent"
ports:
- name: schema-registry
containerPort: 8081
protocol: TCP
- containerPort: 5555
name: jmx
resources:
{}
env:
- name: SCHEMA_REGISTRY_HOST_NAME
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: SCHEMA_REGISTRY_LISTENERS
value: http://0.0.0.0:8081
- name: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS
value: PLAINTEXT://my-dep-apps-cp-kafka-headless:9092
- name: SCHEMA_REGISTRY_KAFKASTORE_GROUP_ID
value: my-dep-apps
- name: SCHEMA_REGISTRY_MASTER_ELIGIBILITY
value: "true"
- name: JMX_PORT
value: "5555"
volumes:
- name: jmx-config
configMap:
name: my-dep-apps-cp-schema-registry-jmx-configmap
---
apiVersion: v1
kind: ConfigMap
metadata:
name: my-dep-apps-cp-ksql-server-jmx-configmap
labels:
app: cp-ksql-server
chart: cp-ksql-server-0.1.0
release: my-dep-apps
heritage: Tiller
data:
jmx-ksql-server-prometheus.yml: |+
jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
lowercaseOutputName: true
lowercaseOutputLabelNames: true
ssl: false
rules:
- pattern : 'io.confluent.ksql.metrics<type=ksql-engine-query-stats>([^:]+):'
name: "cp_ksql_server_metrics_$1"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: my-dep-apps-cp-zookeeper-jmx-configmap
labels:
app: cp-zookeeper
chart: cp-zookeeper-0.1.0
release: my-dep-apps
heritage: Tiller
data:
jmx-zookeeper-prometheus.yml: |+
jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
lowercaseOutputName: true
lowercaseOutputLabelNames: true
ssl: false
rules:
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+)><>(\\w+)"
name: "cp_zookeeper_$2"
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+)><>(\\w+)"
name: "cp_zookeeper_$3"
labels:
replicaId: "$2"
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+)><>(\\w+)"
name: "cp_zookeeper_$4"
labels:
replicaId: "$2"
memberType: "$3"
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+), name3=(\\w+)><>(\\w+)"
name: "cp_zookeeper_$4_$5"
labels:
replicaId: "$2"
memberType: "$3"
---
apiVersion: v1
kind: Service
metadata:
name: my-dep-apps-cp-zookeeper-headless
labels:
app: cp-zookeeper
chart: cp-zookeeper-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
ports:
- port: 2888
name: server
- port: 3888
name: leader-election
clusterIP: None
selector:
app: cp-zookeeper
release: my-dep-apps
---
apiVersion: v1
kind: Service
metadata:
name: my-dep-apps-cp-kafka
labels:
app: cp-kafka
chart: cp-kafka-0.1.0
release: my-dep-apps
heritage: Tiller
spec:
ports:
- port: 9092
name: broker
selector:
app: cp-kafka
release: my-dep-apps
---
apiVersion: v1
kind: ConfigMap
metadata:
name: my-dep-apps-cp-kafka-connect-jmx-configmap
labels:
app: cp-kafka-connect
chart: cp-kafka-connect-0.1.0
release: my-dep-apps
heritage: Tiller
data:
jmx-kafka-connect-prometheus.yml: |+
jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
lowercaseOutputName: true
lowercaseOutputLabelNames: true
ssl: false
rules:
- pattern : "kafka.connect<type=connect-worker-metrics>([^:]+):"
name: "cp_kafka_connect_connect_worker_metrics_$1"
- pattern : "kafka.connect<type=connect-metrics, client-id=([^:]+)><>([^:]+)"
name: "cp_kafka_connect_connect_metrics_$1_$2"
- pattern : "kafka.connect<type=connector-task-metrics, connector=([^:]+), task=([^:]+)><>status: ([^:]+)"
name: "cp_kafka_connect_connect_connector_metrics"
value: 1
labels:
connector: $1
task: $2
status: $3
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment