Kafka behind traefik with kafdrop and kafka-exporter for prometheus

readme.MD

Kafdrop + Kafka + zookeeper behind Traefik Proxy

Make folder before start this compose

mkdir /home/docker/vol/kafka
mkdir /home/docker/vol/zookeeper
chown 1001:1001 /home/docker/vol/kafka
chown 1001:1001 /home/docker/vol/zookeeper

Warning, missing auth on Kafdrop.
Producer and consumer must be in same network pg_kafka

kafka-compose.yml

version: "3.7"
services:
  my-producer:
    container_name: my-producer
    image: my-producer:1.0.0
    restart: "no"
    volumes:
      - /home/docker/vol/my-producer:/logs
    networks:
      - traefik
      - nw_kafka
    environment:
      - MY_PRODUCER_LOG_PROFILE=dev
      - MY_PRODUCER_LOG_CONFIG_FILE=/logs/my-producer.log
      - MY_PRODUCER_AUTH_POST_EVENT_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
      - MY_PRODUCER_KAFKA_BROKER=kafka-broker:9092
      - MY_PRODUCER_KAFKA_TOPIC=MyTopic
    depends_on:
      - "kafka-broker"
    labels:
      - "traefik.http.routers.my-producer.rule=Host(`sub.domain.tld`) && PathPrefix(`\/producer\/`) "
      - "traefik.http.middlewares.my-producer-stripprefix.stripprefix.prefixes=/producer"
      - "traefik.http.routers.my-producer.tls=true"
      - "traefik.http.routers.my-producer.tls.certresolver=letsencrypt"
      - "traefik.http.routers.my-producer.entrypoints=websecure"
      - "traefik.http.routers.my-producer.middlewares=my-producer-stripprefix"
      - "traefik.http.services.my-producer.loadbalancer.server.port=8080"
      - "traefik.http.routers.my-producer.service=my-producer"
      - "traefik.http.routers.my-producer-http.rule=Host(`sub.domain.tld`) && PathPrefix(`\/producer\/`)"
      - "traefik.http.routers.my-producer-http.middlewares=https-redirect@file"
      - "traefik.docker.network=traefik"
      - "traefik.enable=true"
      - "com.centurylinklabs.watchtower.enable=true"


  kafdrop:
    container_name: kafdrop
    image: obsidiandynamics/kafdrop:3.27.0
    restart: "no"
    networks:
      - traefik
      - nw_kafka
    environment:
      - KAFKA_BROKERCONNECT=kafka-broker:9092
      - JVM_OPTS=-Xms32M -Xmx64M
      - SERVER_SERVLET_CONTEXTPATH=/kafdrop
    depends_on:
      - "kafka-broker"
    labels:
      - "traefik.http.routers.kafdrop.rule=Host(`sub.domain.tld`) && PathPrefix(`\/kafdrop\/`) "
      - "traefik.http.routers.kafdrop.tls=true"
      - "traefik.http.routers.kafdrop.tls.certresolver=letsencrypt"
      - "traefik.http.routers.kafdrop.entrypoints=websecure"
      - "traefik.http.routers.kafdrop.middlewares=auth@file"
      - "traefik.http.services.kafdrop.loadbalancer.server.port=9000"
      - "traefik.http.routers.kafdrop.service=kafdrop"
      - "traefik.http.routers.kafdrop-http.rule=Host(`sub.domain.tld`) && PathPrefix(`\/kafdrop\/`)"
      - "traefik.http.routers.kafdrop-http.middlewares=https-redirect@file"
      - "traefik.docker.network=traefik"
      - "traefik.enable=true"

  #https://github.com/danielqsj/kafka_exporter#flags
  kafka-exporter:
    container_name: kafka-exporter
    image: bitnami/kafka-exporter:1.2.0
    restart: always
    command:
#      --web.listen-address :9308
       --web.telemetry-path /kafka-exporter/metrics
       --kafka.server kafka-broker:9092
    networks:
      - traefik
      - nw_kafka
    depends_on:
      - "kafka-broker"
    labels:
      - "traefik.http.routers.kafka-exporter.rule=Host(`sub.domain.tld`) && PathPrefix(`\/kafka-exporter\/`) "
      - "traefik.http.routers.kafka-exporter.tls=true"
      - "traefik.http.routers.kafka-exporter.tls.certresolver=letsencrypt"
      - "traefik.http.routers.kafka-exporter.entrypoints=websecure"
      - "traefik.http.routers.kafka-exporter.middlewares=auth@file"
      - "traefik.http.services.kafka-exporter.loadbalancer.server.port=9308"
      - "traefik.http.routers.kafka-exporter.service=kafka-exporter"
      - "traefik.http.routers.kafka-exporter-http.rule=Host(`sub.domain.tld`) && PathPrefix(`\/kafka-exporter\/`)"
      - "traefik.http.routers.kafka-exporter-http.middlewares=https-redirect@file"
      - "traefik.docker.network=traefik"
      - "traefik.enable=true"

  kafka-broker:
    container_name: kafka-broker
    image: bitnami/kafka:2.6.0
    networks:
      - nw_kafka
    volumes:
      - /home/docker/vol/kafka:/bitnami/kafka
    environment:
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_BROKER_ID=1
      - KAFKA_LISTENERS=PLAINTEXT://:9092
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-broker:9092
    labels:
      - "traefik.tcp.routers.kafka-broker.service=kafka-broker"
      - "traefik.tcp.services.kafka-broker.loadbalancer.server.port=9092"
      - "traefik.enable=false"

  zookeeper:
    container_name: zookeeper
    image: bitnami/zookeeper:3.6.2 #https://hub.docker.com/r/bitnami/zookeeper
    networks:
     - nw_kafka
    volumes:
      - /home/docker/vol/zookeeper:/bitnami/zookeeper
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
      - ZOO_SERVER_ID=1
      - ZOO_PORT_NUMBER=2181
      - ZOO_SERVERS=zookeeper:2888:3888
    labels:
      - "traefik.tcp.routers.zookeeper.service=zookeeper"
      - "traefik.tcp.services.zookeeper.loadbalancer.server.port=2181"
      - "traefik.enable=false"

networks:
  nw_kafka:
    name: nw_kafka
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: 172.21.00.96/28