MirrorMaker 2.0 (based on Kafka 2.7) for Strimzi

mirrormaker2-plaintext-with-metrics.yaml

apiVersion: kafka.strimzi.io/v1alpha1
kind: KafkaMirrorMaker2
metadata:
  name: my-mirror-maker-2
spec:
  version: 2.6.0
  replicas: 1
  # Custom Image that includes MirrorMaker 2.0 (based on Kafka 2.7 SNAPSHOT) for offset sync & translation
  image: <custom_strimzi_image_based_on_Kafka_2.7_bin>
  connectCluster: "msk"
  clusters:
  - alias: opsworks
    bootstrapServers: "<OPSWORKS BROKER LIST ON PLAINTEXT>""
  - alias: msk
    bootstrapServers: "<MSK BROKER LIST ON PLAINTEXT>""
    config:
      config.storage.replication.factor: 3
      offset.storage.replication.factor: 3
      status.storage.replication.factor: 3
  mirrors:
  - sourceCluster: "opsworks"
    targetCluster: "msk"
    sourceConnector:
      tasksMax: 10
      config:
        replication.factor: 3
        offset-syncs.topic.replication.factor: 3
        sync.topic.acls.enabled: "false"
        replication.policy.separator: ""
        replication.policy.class: "io.strimzi.kafka.connect.mirror.IdentityReplicationPolicy"
        refresh.groups.interval.seconds: 5
        refresh.topics.interval.seconds: 5
        sync.group.offsets.enabled: true
        sync.group.offsets.interval.seconds: 5
    heartbeatConnector:
      config:
        heartbeats.topic.replication.factor: 3
    checkpointConnector:
      tasksMax: 10
      config:
        sync.topic.acls.enabled: "false"
        checkpoints.topic.replication.factor: 3
        refresh.groups.interval.seconds: 5
        refresh.topics.interval.seconds: 5
        emit.checkpoints.interval.seconds: 5
        sync.group.offsets.enabled: true
        sync.group.offsets.interval.seconds: 5
        replication.policy.separator: ""
        replication.policy.class: "io.strimzi.kafka.connect.mirror.IdentityReplicationPolicy"
    topicsPattern: ".*"
    groupsPattern: ".*"
    groupsBlacklistPattern: ""
  resources:
    requests:
      cpu: "4"
      memory: 5Gi
    limits:
      cpu: "8"
      memory: 10Gi
#   logging:
#     type: inline
#     loggers:
#       connect.root.logger.level: "DEBUG"
  metrics:
    lowercaseOutputLabelNames: true
    lowercaseOutputName: true
    rules:
    - help: Kafka $1 JMX metric start time seconds
      labels:
        clientId: $2
      name: kafka_$1_start_time_seconds
      pattern: kafka.(.+)<type=app-info, client-id=(.+)><>start-time-ms
      type: GAUGE
      valueFactor: 0.001
    - help: Kafka $1 JMX metric info version and commit-id
      labels:
        $3: $4
        clientId: $2
      name: kafka_$1_$3_info
      pattern: 'kafka.(.+)<type=app-info, client-id=(.+)><>(commit-id|version): (.+)'
      type: GAUGE
      value: 1
    - help: Kafka $1 JMX metric type $2
      labels:
        clientId: $3
        partition: $5
        topic: $4
      name: kafka_$2_$6
      pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+), partition=(.+)><>(.+-total|compression-rate|.+-avg|.+-replica|.+-lag|.+-lead)
      type: GAUGE
    - help: Kafka $1 JMX metric type $2
      labels:
        clientId: $3
        topic: $4
      name: kafka_$2_$5
      pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), topic=(.+)><>(.+-total|compression-rate|.+-avg)
      type: GAUGE
    - help: Kafka $1 JMX metric type $2
      labels:
        clientId: $3
        nodeId: $4
      name: kafka_$2_$5
      pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.+), node-id=(.+)><>(.+-total|.+-avg)
      type: UNTYPED
    - help: Kafka $1 JMX metric type $2
      labels:
        clientId: $3
      name: kafka_$2_$4
      pattern: kafka.(.+)<type=(.+)-metrics, client-id=(.*)><>(.+-total|.+-avg|.+-bytes|.+-count|.+-ratio|.+-age|.+-flight|.+-threads|.+-connectors|.+-tasks|.+-ago)
      type: GAUGE
    - help: Kafka Connect JMX Connector status
      labels:
        connector: $1
        status: $3
        task: $2
      name: kafka_connect_connector_status
      pattern: 'kafka.connect<type=connector-task-metrics, connector=(.+), task=(.+)><>status:
        ([a-z-]+)'
      type: GAUGE
      value: 1
    - help: Kafka Connect JMX metric type $1
      labels:
        connector: $2
        task: $3
      name: kafka_connect_$1_$4
      pattern: kafka.connect<type=(.+)-metrics, connector=(.+), task=(.+)><>(.+-total|.+-count|.+-ms|.+-ratio|.+-avg|.+-failures|.+-requests|.+-timestamp|.+-logged|.+-errors|.+-retries|.+-skipped)
      type: GAUGE
    - help: Kafka Connect JMX metric $1
      labels:
        connector: $1
      name: kafka_connect_worker_$2
      pattern: kafka.connect<type=connect-worker-metrics, connector=(.+)><>([a-z-]+)
      type: GAUGE
    - help: Kafka Connect JMX metric worker
      name: kafka_connect_worker_$1
      pattern: kafka.connect<type=connect-worker-metrics><>([a-z-]+)
      type: GAUGE
    - help: Kafka Connect JMX metric rebalance information
      name: kafka_connect_worker_rebalance_$1
      pattern: kafka.connect<type=connect-worker-rebalance-metrics><>([a-z-]+)
      type: GAUGE
    # If you need all MirrorMaker 2.0 Metrics exported then you should enable the following 2 Patterns & Comment Out the next 2
    # - pattern: kafka.connect.mirror<type=MirrorSourceConnector, target=(.+), topic=(.+), partition=(.+)><>([a-z-]+)
    #   name: kafka_connect_mirror_$4
    #   labels:
    #     target: $1
    #     topic: $2
    #     partition: $3
    #   help: "Kafka Connect Mirror JMX metric $4"
    #   type: GAUGE
    # - pattern: kafka.connect.mirror<type=MirrorCheckpointConnector, source=(.+), target=(.+), group=(.+), topic=(.+), partition=(.+)><>([a-z-]+)
    #   name: kafka_connect_mirror_$6
    #   labels:
    #     source: $1
    #     target: $2
    #     group: $3
    #     topic: $4
    #     partition: $5
    #   help: "Kafka Connect Mirror JMX metric $6"
    #   type: GAUGE
    - pattern: kafka.connect.mirror<type=MirrorSourceConnector, target=(.+), topic=(.+), partition=(.+)><>(.+-ms-avg)
      name: kafka_connect_mirror_$4
      labels:
        target: $1
        topic: $2
        partition: $3
      help: "Kafka Connect Mirror JMX metric $4"
      type: GAUGE
    - pattern: kafka.connect.mirror<type=MirrorCheckpointConnector, source=(.+), target=(.+), group=(.+), topic=(.+), partition=(.+)><>(.+-avg)
      name: kafka_connect_mirror_$6
      labels:
        source: $1
        target: $2
        group: $3
        topic: $4
      help: "Kafka Connect Mirror JMX metric $6"
      type: GAUGE