Created
October 25, 2019 20:36
-
-
Save jhutar/949692e775ed48433b9abc267deeb31c to your computer and use it in GitHub Desktop.
Slighlty modified https://github.com/rediverson/streams-perf/blob/master/examples/kafka/kafka-perftest-m52xlarge.yaml to fit to m5.xlarge
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apiVersion: kafka.strimzi.io/v1alpha1 | |
kind: Kafka | |
metadata: | |
name: my-cluster | |
labels: | |
app: my-cluster | |
spec: | |
kafka: | |
replicas: 3 | |
resources: | |
requests: | |
memory: 10Gi | |
cpu: 2 | |
limits: | |
memory: 10Gi | |
cpu: 2 | |
jvmOptions: | |
-Xms: 2048m | |
-Xmx: 2048m | |
listeners: | |
plain: {} | |
tls: {} | |
external: | |
type: route | |
config: | |
auto.create.topics.enable: "false" | |
offsets.topic.replication.factor: 3 | |
transaction.state.log.replication.factor: 3 | |
transaction.state.log.min.isr: 2 | |
storage: | |
type: persistent-claim | |
size: 1000Gi | |
deleteClaim: true | |
rack: | |
topologyKey: failure-domain.beta.kubernetes.io/zone | |
metrics: | |
# Inspired by config from Kafka 2.0.0 example rules: | |
# https://github.com/prometheus/jmx_exporter/blob/master/example_configs/kafka-2_0_0.yml | |
lowercaseOutputName: true | |
rules: | |
# Special cases and very specific rules | |
- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value | |
name: kafka_server_$1_$2 | |
type: GAUGE | |
labels: | |
clientId: "$3" | |
topic: "$4" | |
partition: "$5" | |
- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value | |
name: kafka_server_$1_$2 | |
type: GAUGE | |
labels: | |
clientId: "$3" | |
broker: "$4:$5" | |
# Some percent metrics use MeanRate attribute | |
# Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate | |
- pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate | |
name: kafka_$1_$2_$3_percent | |
type: GAUGE | |
# Generic gauges for percents | |
- pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value | |
name: kafka_$1_$2_$3_percent | |
type: GAUGE | |
- pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value | |
name: kafka_$1_$2_$3_percent | |
type: GAUGE | |
labels: | |
"$4": "$5" | |
# Generic per-second counters with 0-2 key/value pairs | |
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count | |
name: kafka_$1_$2_$3_total | |
type: COUNTER | |
labels: | |
"$4": "$5" | |
"$6": "$7" | |
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count | |
name: kafka_$1_$2_$3_total | |
type: COUNTER | |
labels: | |
"$4": "$5" | |
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count | |
name: kafka_$1_$2_$3_total | |
type: COUNTER | |
# Generic gauges with 0-2 key/value pairs | |
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value | |
name: kafka_$1_$2_$3 | |
type: GAUGE | |
labels: | |
"$4": "$5" | |
"$6": "$7" | |
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value | |
name: kafka_$1_$2_$3 | |
type: GAUGE | |
labels: | |
"$4": "$5" | |
- pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value | |
name: kafka_$1_$2_$3 | |
type: GAUGE | |
# Emulate Prometheus 'Summary' metrics for the exported 'Histogram's. | |
# Note that these are missing the '_sum' metric! | |
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count | |
name: kafka_$1_$2_$3_count | |
type: COUNTER | |
labels: | |
"$4": "$5" | |
"$6": "$7" | |
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile | |
name: kafka_$1_$2_$3 | |
type: GAUGE | |
labels: | |
"$4": "$5" | |
"$6": "$7" | |
quantile: "0.$8" | |
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count | |
name: kafka_$1_$2_$3_count | |
type: COUNTER | |
labels: | |
"$4": "$5" | |
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile | |
name: kafka_$1_$2_$3 | |
type: GAUGE | |
labels: | |
"$4": "$5" | |
quantile: "0.$6" | |
- pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count | |
name: kafka_$1_$2_$3_count | |
type: COUNTER | |
- pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile | |
name: kafka_$1_$2_$3 | |
type: GAUGE | |
labels: | |
quantile: "0.$4" | |
zookeeper: | |
replicas: 3 | |
resources: | |
requests: | |
memory: 2Gi | |
cpu: 0.7 | |
limits: | |
memory: 2Gi | |
cpu: 0.7 | |
jvmOptions: | |
-Xms: 512m | |
-Xmx: 512m | |
storage: | |
type: persistent-claim | |
size: 100Gi | |
deleteClaim: true | |
metrics: | |
# Inspired by Zookeeper rules | |
# https://github.com/prometheus/jmx_exporter/blob/master/example_configs/zookeeper.yaml | |
lowercaseOutputName: true | |
rules: | |
# replicated Zookeeper | |
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+)><>(\\w+)" | |
name: "zookeeper_$2" | |
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+)><>(\\w+)" | |
name: "zookeeper_$3" | |
labels: | |
replicaId: "$2" | |
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+)><>(\\w+)" | |
name: "zookeeper_$4" | |
labels: | |
replicaId: "$2" | |
memberType: "$3" | |
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+), name3=(\\w+)><>(\\w+)" | |
name: "zookeeper_$4_$5" | |
labels: | |
replicaId: "$2" | |
memberType: "$3" | |
# standalone Zookeeper | |
- pattern: "org.apache.ZooKeeperService<name0=StandaloneServer_port(\\d+)><>(\\w+)" | |
name: "zookeeper_$2" | |
- pattern: "org.apache.ZooKeeperService<name0=StandaloneServer_port(\\d+), name1=(InMemoryDataTree)><>(\\w+)" | |
name: "zookeeper_$2_$3" | |
entityOperator: | |
topicOperator: | |
resources: | |
requests: | |
memory: 256Mi | |
cpu: 500m | |
limits: | |
memory: 256Mi | |
cpu: 500m | |
# logging: | |
# loggers: | |
# rootLogger.level: INFO | |
# type: inline | |
userOperator: | |
resources: | |
requests: | |
memory: 256Mi | |
cpu: "0.2" | |
limits: | |
memory: 256Mi | |
cpu: "0.5" | |
# logging: | |
# loggers: | |
# rootLogger.level: INFO | |
# type: inline |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment