Kafka with persistent storage - Template OCP

barnabas-persistent.yml

apiVersion: template.openshift.io/v1
kind: Template
metadata:
  annotations:
    description: This template installes Apache Zookeeper and Apache Kafka clusters.
      For more information about using this template see https://github.com/EnMasseProject/barnabas
    iconClass: fa pficon-topology
    openshift.io/display-name: Apache Kafka (Persistent storage)
    tags: messaging,datastore
    template.openshift.io/documentation-url: https://github.com/EnMasseProject/barnabas
  creationTimestamp: null
  name: barnabas-persistent
  selfLink: /apis/template.openshift.io/v1/namespaces/openshift/templates/barnabas-persistent
objects:
- apiVersion: v1
  kind: Service
  metadata:
    name: kafka
  spec:
    ports:
    - name: kafka
      port: 9092
      protocol: TCP
      targetPort: 9092
    selector:
      name: kafka
    type: ClusterIP
- apiVersion: v1
  kind: Service
  metadata:
    name: kafka-headless
  spec:
    clusterIP: None
    ports:
    - name: kafka
      port: 9092
      protocol: TCP
      targetPort: 9092
    selector:
      name: kafka
    type: ClusterIP
- apiVersion: v1
  kind: Service
  metadata:
    name: zookeeper
  spec:
    ports:
    - name: clientport
      port: 2181
      protocol: TCP
      targetPort: 2181
    selector:
      name: zookeeper
    type: ClusterIP
- apiVersion: v1
  kind: Service
  metadata:
    name: zookeeper-headless
  spec:
    clusterIP: None
    ports:
    - name: clientport
      port: 2181
      protocol: TCP
      targetPort: 2181
    - name: clustering
      port: 2888
      protocol: TCP
      targetPort: 2888
    - name: leaderelection
      port: 3888
      protocol: TCP
      targetPort: 3888
    selector:
      name: zookeeper
    type: ClusterIP
- apiVersion: v1
  data:
    config.yml: |
      lowercaseOutputName: true
      rules:
      - pattern: kafka.server<type=(.+), name=(.+)PerSec\w*><>Count
        name: kafka_server_$1_$2_total
      - pattern : kafka.server<type=(.+), name=(.+)PerSec\w*, topic=(.+)><>Count
        name: kafka_server_$1_$2_total
        labels:
          topic: "$3"
  kind: ConfigMap
  metadata:
    name: kafka-metrics-config
- apiVersion: apps/v1beta1
  kind: StatefulSet
  metadata:
    labels:
      name: kafka
    name: kafka
  spec:
    replicas: ${KAFKA_NODE_COUNT}
    selector:
      matchLabels:
        name: kafka
    serviceName: kafka-headless
    template:
      metadata:
        annotations:
          prometheus.io/path: /metrics
          prometheus.io/port: "9404"
          prometheus.io/scrape: "true"
        labels:
          name: kafka
      spec:
        containers:
        - env:
          - name: KAFKA_DEFAULT_REPLICATION_FACTOR
            value: ${KAFKA_DEFAULT_REPLICATION_FACTOR}
          - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
            value: ${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR}
          - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
            value: ${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR}
          - name: KAFKA_METRICS_ENABLED
            value: ${KAFKA_METRICS_ENABLED}
          image: ${IMAGE_REPO_NAME}/${KAFKA_IMAGE_NAME}:${KAFKA_IMAGE_TAG}
          imagePullPolicy: Always
          livenessProbe:
            exec:
              command:
              - /opt/kafka/kafka_healthcheck.sh
            initialDelaySeconds: ${ZOOKEEPER_HEALTHCHECK_DELAY}
            timeoutSeconds: ${ZOOKEEPER_HEALTHCHECK_TIMEOUT}
          name: kafka
          ports:
          - containerPort: 9092
            name: kafka
            protocol: TCP
          - containerPort: 9404
            name: kafkametrics
            protocol: TCP
          readinessProbe:
            exec:
              command:
              - /opt/kafka/kafka_healthcheck.sh
            initialDelaySeconds: ${ZOOKEEPER_HEALTHCHECK_DELAY}
            timeoutSeconds: ${ZOOKEEPER_HEALTHCHECK_TIMEOUT}
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /var/lib/kafka/
            name: kafka-storage
          - mountPath: /opt/prometheus/config/
            name: kafka-metrics-config
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        volumes:
        - configMap:
            name: kafka-metrics-config
          name: kafka-metrics-config
    volumeClaimTemplates:
    - metadata:
        name: kafka-storage
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: ${KAFKA_VOLUME_CAPACITY}
- apiVersion: v1
  data:
    config.yml: |
      lowercaseOutputName: true
  kind: ConfigMap
  metadata:
    name: zookeeper-metrics-config
- apiVersion: apps/v1beta1
  kind: StatefulSet
  metadata:
    labels:
      name: zookeeper
    name: zookeeper
  spec:
    replicas: ${ZOOKEEPER_NODE_COUNT}
    selector:
      matchLabels:
        name: zookeeper
    serviceName: zookeeper-headless
    template:
      metadata:
        annotations:
          prometheus.io/path: /metrics
          prometheus.io/port: "9404"
          prometheus.io/scrape: "true"
        labels:
          name: zookeeper
      spec:
        containers:
        - env:
          - name: ZOOKEEPER_NODE_COUNT
            value: ${ZOOKEEPER_NODE_COUNT}
          - name: ZOOKEEPER_METRICS_ENABLED
            value: ${ZOOKEEPER_METRICS_ENABLED}
          image: ${IMAGE_REPO_NAME}/${ZOOKEEPER_IMAGE_NAME}:${ZOOKEEPER_IMAGE_TAG}
          imagePullPolicy: Always
          livenessProbe:
            exec:
              command:
              - /opt/zookeeper/zookeeper_healthcheck.sh
            initialDelaySeconds: ${KAFKA_HEALTHCHECK_DELAY}
            timeoutSeconds: ${KAFKA_HEALTHCHECK_TIMEOUT}
          name: zookeeper
          ports:
          - containerPort: 2181
            name: clientport
            protocol: TCP
          - containerPort: 2888
            name: clustering
            protocol: TCP
          - containerPort: 3888
            name: leaderelection
            protocol: TCP
          - containerPort: 9404
            name: kafkametrics
            protocol: TCP
          readinessProbe:
            exec:
              command:
              - /opt/zookeeper/zookeeper_healthcheck.sh
            initialDelaySeconds: ${KAFKA_HEALTHCHECK_DELAY}
            timeoutSeconds: ${KAFKA_HEALTHCHECK_TIMEOUT}
          volumeMounts:
          - mountPath: /var/lib/zookeeper
            name: zookeeper-storage
          - mountPath: /opt/prometheus/config/
            name: zookeeper-metrics-config
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        volumes:
        - configMap:
            name: zookeeper-metrics-config
          name: zookeeper-metrics-config
    volumeClaimTemplates:
    - metadata:
        name: zookeeper-storage
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: ${ZOOKEEPER_VOLUME_CAPACITY}
parameters:
- description: Number of Zookeper cluster nodes which will be deployed (odd number
    of nodes is recomended)
  displayName: Number of Zookeper cluster nodes (odd number of nodes is recomended)
  name: ZOOKEEPER_NODE_COUNT
  required: true
  value: "3"
- description: Number of Kafka cluster nodes which will be deployed
  displayName: Number of Kafka cluster nodes
  name: KAFKA_NODE_COUNT
  required: true
  value: "3"
- description: Volume space available for Zookeeper data, e.g. 512Mi, 2Gi.
  displayName: Zookeeper Volume Capacity
  name: ZOOKEEPER_VOLUME_CAPACITY
  required: true
  value: 1Gi
- description: Volume space available for Kafka data, e.g. 512Mi, 2Gi.
  displayName: Kafka Volume Capacity
  name: KAFKA_VOLUME_CAPACITY
  required: true
  value: 1Gi
- description: Image repository name
  displayName: Repository Name
  name: IMAGE_REPO_NAME
  value: enmasseproject
- description: Kafka image name
  displayName: Kafka image Name
  name: KAFKA_IMAGE_NAME
  value: kafka-statefulsets
- description: Kafka image tag
  displayName: Kafka image tag
  name: KAFKA_IMAGE_TAG
  value: latest
- description: Zookeeper image name
  displayName: Zookeeper image Name
  name: ZOOKEEPER_IMAGE_NAME
  value: zookeeper
- description: Zookeeper image tag
  displayName: Zookeeper image tag
  name: ZOOKEEPER_IMAGE_TAG
  value: latest
- description: Number of seconds after the container has started before healthcheck
    probes are initiated.
  displayName: Zookeeper healthcheck initial delay
  name: ZOOKEEPER_HEALTHCHECK_DELAY
  value: "15"
- description: Number of seconds after which the probe times out.
  displayName: Zookeeper healthcheck timeout
  name: ZOOKEEPER_HEALTHCHECK_TIMEOUT
  value: "5"
- description: Number of seconds after the container has started before healthcheck
    probes are initiated.
  displayName: Kafka healthcheck initial delay
  name: KAFKA_HEALTHCHECK_DELAY
  value: "15"
- description: Number of seconds after which the probe times out.
  displayName: Kafka healthcheck timeout
  name: KAFKA_HEALTHCHECK_TIMEOUT
  value: "5"
- description: Default replication factor for newly created topics
  displayName: Default replication factor
  name: KAFKA_DEFAULT_REPLICATION_FACTOR
  value: "1"
- description: Replication factor for offsets topic
  displayName: Offsets replication factor
  name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
  value: "3"
- description: Replication factor for transactions state log topic
  displayName: Transaction state replication factor
  name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
  value: "3"
- description: true to enable metric collection from Kafka
  displayName: Kafka metrics enabled
  name: KAFKA_METRICS_ENABLED
  value: "true"
- description: true to enable metric collection from Zookeeper
  displayName: Zookeeper metrics enabled
  name: ZOOKEEPER_METRICS_ENABLED
  value: "true"