Skip to content

Instantly share code, notes, and snippets.

@b13n1u
Created October 5, 2018 11:51
Show Gist options
  • Save b13n1u/8628bdf9601a9fbe2588b655b31ebde2 to your computer and use it in GitHub Desktop.
Save b13n1u/8628bdf9601a9fbe2588b655b31ebde2 to your computer and use it in GitHub Desktop.
Not so perfect collectd genericjmx plugin config for kafka mirrormaker.
# Not so perfect collectd generic jmx plugin config for kafka mirrormaker
# You need to enable JMX for mirrormaker first.
# Documentation:
# https://collectd.org/wiki/index.php/Plugin:Java
<Plugin java>
<Plugin "GenericJMX">
<MBean "classes">
ObjectName "java.lang:type=ClassLoading"
<Value>
Type "gauge"
InstancePrefix "loaded_classes"
Table false
Attribute "LoadedClassCount"
</Value>
</MBean>
<MBean "garbage_collector">
ObjectName "java.lang:type=GarbageCollector,*"
InstancePrefix ".gc."
InstanceFrom "name"
<Value>
Type "invocations"
#InstancePrefix ""
#InstanceFrom ""
Table false
Attribute "CollectionCount"
</Value>
<Value>
Type "total_time_in_ms"
InstancePrefix "collection_time"
#InstanceFrom ""
Table false
Attribute "CollectionTime"
</Value>
</MBean>
<MBean "memory-heap">
ObjectName "java.lang:type=Memory"
#InstanceFrom ""
InstancePrefix ".memory-heap."
# Creates four values: committed, init, max, used
<Value>
Type "jmx_memory"
#InstancePrefix ""
#InstanceFrom ""
Table true
Attribute "HeapMemoryUsage"
</Value>
</MBean>
# Non-heap memory usage
<MBean "memory-nonheap">
ObjectName "java.lang:type=Memory"
#InstanceFrom ""
InstancePrefix ".memory-nonheap."
# Creates four values: committed, init, max, used
<Value>
Type "jmx_memory"
#InstancePrefix ""
#InstanceFrom ""
Table true
Attribute "NonHeapMemoryUsage"
</Value>
</MBean>
<MBean "memory_pool">
ObjectName "java.lang:type=MemoryPool,*"
InstancePrefix ".memory_pool."
InstanceFrom "name"
<Value>
Type "jmx_memory"
#InstancePrefix ""
#InstanceFrom ""
Table true
Attribute "Usage"
</Value>
</MBean>
<MBean "kafka_tools">
ObjectName "kafka.tools:type=MirrorMaker,name=MirrorMaker-numDroppedMessages"
InstancePrefix ".mirrormaker."
InstanceFrom "name"
<Value>
Type "counter"
#InstancePrefix ""
#InstanceFrom ""
Table false
Attribute "Value"
</Value>
</MBean>
<MBean "kafka_consumer_app_info">
ObjectName "kafka.consumer:type=app-info,client-id=*"
InstancePrefix ".consumer.app-info."
InstanceFrom "client-id"
<Value>
Type "gauge"
#InstancePrefix ""
#InstanceFrom ""
Table false
Attribute "version"
</Value>
</MBean>
<MBean "kafka_consumer_coordinator-metrics">
ObjectName "kafka.consumer:type=consumer-coordinator-metrics,client-id=*"
InstancePrefix ".consumer.coordinator."
InstanceFrom "client-id"
<Value>
Type "gauge"
InstancePrefix"join-time-max"
Table false
Attribute "join-time-max"
</Value>
<Value>
InstancePrefix "commit-latency-avg"
Table false
Attribute "commit-latency-avg"
Type "gauge"
</Value>
<Value>
InstancePrefix "sync-time-avg"
Table false
Attribute "sync-time-avg"
Type "gauge"
</Value>
<Value>
InstancePrefix "assigned-partitions"
Table false
Attribute "assigned-partitions"
Type "count"
</Value>
<Value>
InstancePrefix "sync-rate"
Table false
Attribute "sync-rate"
Type "count"
</Value>
<Value>
InstancePrefix "commit-rate"
Table false
Attribute "commit-rate"
Type "count"
</Value>
<Value>
InstancePrefix "last-heartbeat-seconds-ago"
Table false
Attribute "last-heartbeat-seconds-ago"
Type "count"
</Value>
<Value>
InstancePrefix "heartbeat-rate"
Table false
Attribute "heartbeat-rate"
Type "count"
</Value>
<Value>
InstancePrefix "commit-latency-max"
Table false
Attribute "commit-latency-max"
Type "gauge"
</Value>
<Value>
InstancePrefix "join-time-avg"
Table false
Attribute "join-time-avg"
Type "gauge"
</Value>
<Value>
InstancePrefix "sync-total"
Table false
Attribute "sync-total"
Type "count"
</Value>
<Value>
InstancePrefix "sync-time-max"
Table false
Attribute "sync-time-max"
Type "count"
</Value>
<Value>
InstancePrefix "join-total"
Table false
Attribute "join-total"
Type "count"
</Value>
<Value>
InstancePrefix "heartbeat-response-time-max"
Table false
Attribute "heartbeat-response-time-max"
Type "gauge"
</Value>
</MBean>
<MBean "kafka_consumer_fetch_manager_metrics">
ObjectName "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*"
InstancePrefix ".consumer.fetch-manager."
InstanceFrom "client-id"
<Value>
InstancePrefix ".bytes-consumed-total."
Attribute "bytes-consumed-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".bytes-consumed-rate."
Attribute "bytes-consumed-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".records-consumed-total."
Attribute "records-consumed-total"
Type "gauge"
</Value>
<Value>
InstancePrefix ".records-consumed-rate."
Attribute "records-consumed-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".records-per-request-avg."
Attribute "records-per-request-avg"
Type "gauge"
</Value>
</MBean>
<MBean "kafka_consumer_fetch_manager_topic_metrics">
ObjectName "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*,topic=*"
InstancePrefix ".consumer.fetch-manager."
InstanceFrom "client-id"
<Value>
InstancePrefix ".bytes-consumed-total."
InstanceFrom "topic"
Attribute "bytes-consumed-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".bytes-consumed-rate."
InstanceFrom "topic"
Attribute "bytes-consumed-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".fetch-size-avg."
InstanceFrom "topic"
Attribute "fetch-size-avg"
Type "gauge"
</Value>
<Value>
InstancePrefix ".fetch-size-max."
InstanceFrom "topic"
Attribute "fetch-size-max"
Type "gauge"
</Value>
</MBean>
<MBean "kafka_consumer_fetch_manager_partition_metrics">
ObjectName "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*,topic=*,partition=*"
InstancePrefix ".consumer.fetch-manager."
InstanceFrom "client-id"
<Value>
InstancePrefix ".records-lag-max."
Table false
InstanceFrom "topic"
InstanceFrom "partition"
Attribute "records-lag-max"
Type "counter"
</Value>
<Value>
InstancePrefix ".records-lag."
InstanceFrom "topic"
InstanceFrom "partition"
Attribute "records-lag"
Type "gauge"
</Value>
<Value>
InstancePrefix ".records-lag-avg."
InstanceFrom "topic"
InstanceFrom "partition"
Attribute "records-lag-avg"
Type "gauge"
</Value>
</MBean>
<MBean "kafka_consumer_metrics">
ObjectName "kafka.consumer:type=consumer-metrics,client-id=*"
InstancePrefix ".consumer."
InstanceFrom "client-id"
<Value>
InstancePrefix ".connection-creation-total."
Table false
Attribute "connection-creation-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".response-rate."
Attribute "response-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".select-rate."
Attribute "select-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".connection-close-total."
Attribute "connection-close-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".network-io-rate."
Attribute "network-io-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".io-ratio."
Attribute "io-ratio"
Type "counter"
</Value>
<Value>
InstancePrefix ".request-total."
Attribute "request-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".io-wait-ratio."
Attribute "io-wait-ratio"
Type "counter"
</Value>
<Value>
InstancePrefix ".outgoing-byte-rate."
Attribute "outgoing-byte-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".request-size-max."
Attribute "request-size-max"
Type "counter"
</Value>
<Value>
InstancePrefix ".network-io-total."
Attribute "network-io-total"
Type "gauge"
</Value>
<Value>
InstancePrefix ".incoming-byte-rate."
Attribute "incoming-byte-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".request-size-avg."
Attribute "request-size-avg"
Type "gauge"
</Value>
<Value>
InstancePrefix ".connection-count."
Attribute "connection-count"
Type "gauge"
</Value>
<Value>
InstancePrefix ".io-waittime-total."
Attribute "io-waittime-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".request-rate."
Attribute "request-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".outgoing-byte-total."
Attribute "outgoing-byte-total"
Type "counter"
</Value>
</MBean>
<MBean "kafka_producer_metrics">
ObjectName "kafka.producer:type=producer-metrics,client-id=*"
InstancePrefix ".producer."
InstanceFrom "client-id"
<Value>
InstancePrefix ".batch-size-avg."
Table false
Attribute "batch-size-avg"
Type "gauge"
</Value>
<Value>
InstancePrefix ".record-error-rate."
Table false
Attribute "record-error-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".record-error-total."
Table false
Attribute "record-error-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".record-retry-rate."
Table false
Attribute "record-retry-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".record-retry-total."
Table false
Attribute "record-retry-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".record-send-rate."
Table false
Attribute "record-send-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".record-send-total."
Table false
Attribute "record-send-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".request-latency-avg."
Table false
Attribute "request-latency-avg"
Type "gauge"
</Value>
<Value>
InstancePrefix ".request-latency-max."
Table false
Attribute "request-latency-max"
Type "gauge"
</Value>
<Value>
InstancePrefix ".buffer-available-bytes."
Table false
Attribute "buffer-available-bytes"
Type "gauge"
</Value>
<Value>
InstancePrefix ".buffer-exhausted-rate."
Table false
Attribute "buffer-exhausted-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".waiting-threads."
Table false
Attribute "waiting-threads"
Type "gauge"
</Value>
<Value>
InstancePrefix ".connection-creation-total."
Table false
Attribute "connection-creation-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".response-rate."
Attribute "response-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".select-rate."
Attribute "select-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".connection-close-total."
Attribute "connection-close-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".network-io-rate."
Attribute "network-io-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".io-ratio."
Attribute "io-ratio"
Type "counter"
</Value>
<Value>
InstancePrefix ".request-total."
Attribute "request-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".io-wait-ratio."
Attribute "io-wait-ratio"
Type "counter"
</Value>
<Value>
InstancePrefix ".outgoing-byte-rate."
Attribute "outgoing-byte-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".request-size-max."
Attribute "request-size-max"
Type "counter"
</Value>
<Value>
InstancePrefix ".network-io-total."
Attribute "network-io-total"
Type "gauge"
</Value>
<Value>
InstancePrefix ".incoming-byte-rate."
Attribute "incoming-byte-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".request-size-avg."
Attribute "request-size-avg"
Type "gauge"
</Value>
<Value>
InstancePrefix ".connection-count."
Attribute "connection-count"
Type "gauge"
</Value>
<Value>
InstancePrefix ".io-waittime-total."
Attribute "io-waittime-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".request-rate."
Attribute "request-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".outgoing-byte-total."
Attribute "outgoing-byte-total"
Type "counter"
</Value>
</MBean>
<MBean "kafka_producer_topic_metrics">
ObjectName "kafka.producer:type=producer-topic-metrics,client-id=*,topic=*"
InstancePrefix ".consumer.fetch-manager."
InstanceFrom "client-id"
<Value>
InstancePrefix ".record-retry-rate."
Table false
InstanceFrom "topic"
Attribute "record-retry-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".record-send-rate."
InstanceFrom "topic"
Attribute "record-send-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".record-retry-total."
InstanceFrom "topic"
Attribute "record-retry-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".compression-rate."
InstanceFrom "topic"
Attribute "compression-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".record-error-total."
InstanceFrom "topic"
Attribute "record-error-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".byte-rate."
InstanceFrom "topic"
Attribute "byte-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".record-error-rate."
InstanceFrom "topic"
Attribute "record-error-rate"
Type "gauge"
</Value>
<Value>
InstancePrefix ".byte-total."
InstanceFrom "topic"
Attribute "byte-total"
Type "counter"
</Value>
<Value>
InstancePrefix ".record-send-total."
InstanceFrom "topic"
Attribute "record-send-total"
Type "counter"
</Value>
</MBean>
<Connection>
ServiceURL "service:jmx:rmi:///jndi/rmi://<%= "localhost:#{port}" %>/jmxrmi"
Host "<%= hostname %>"
InstancePrefix "<%= prefix %>"
Collect "classes"
Collect "garbage_collector"
Collect "memory-heap"
Collect "memory-nonheap"
Collect "memory_pool"
Collect "kafka_tools"
# Collect "kafka_consumer_app_info"
Collect "kafka_consumer_metrics"
Collect "kafka_consumer_coordinator-metrics"
Collect "kafka_consumer_fetch_manager_metrics"
Collect "kafka_consumer_fetch_manager_topic_metrics"
Collect "kafka_consumer_fetch_manager_partition_metrics"
Collect "kafka_producer_topic_metrics"
Collect "kafka_producer_metrics"
</Connection>
</Plugin>
</Plugin>
@syswipe
Copy link

syswipe commented May 27, 2019

I tried to use Mbean named "consumer-fetch-manager-metrics", but i received "warning] GenericJMXConfMBean: No MBean matched the ObjectName kafka.consumer:type=consumer-fetch-manager-metrics,client-id=,topic=,partition=*" warning.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment