Unifi CloudKey Prometheus monitoring

jmx-config.yml

# taken from https://github.com/jacobalberty/unifi-docker/issues/408
---
lowercaseOutputLabelNames: true
lowercaseOutputName: true
rules:
- pattern: 'Tomcat<type=GlobalRequestProcessor, name=\"(\w+-\w+)-(\d+)\"><>(\w+):'
  name: tomcat_$3_total
  labels:
    port: "$2"
    protocol: "$1"
  help: Tomcat global $3
  type: COUNTER
- pattern: 'Tomcat<j2eeType=Servlet, WebModule=//([-a-zA-Z0-9+&@#/%?=~_|!:.,;]*[-a-zA-Z0-9+&@#/%=~_|]), name=([-a-zA-Z0-9+/$%~_-|!.]*), J2EEApplication=none, J2EEServer=none><>(requestCount|maxTime|processingTime|errorCount):'
  name: tomcat_servlet_$3_total
  labels:
    module: "$1"
    servlet: "$2"
  help: Tomcat servlet $3 total
  type: COUNTER
- pattern: 'Tomcat<type=ThreadPool, name="(\w+-\w+)-(\d+)"><>(currentThreadCount|currentThreadsBusy|keepAliveCount|pollerThreadCount|connectionCount):'
  name: tomcat_threadpool_$3
  labels:
    port: "$2"
    protocol: "$1"
  help: Tomcat threadpool $3
  type: GAUGE
- pattern: 'Tomcat<type=Manager, host=([-a-zA-Z0-9+&@#/%?=~_|!:.,;]*[-a-zA-Z0-9+&@#/%=~_|]), context=([-a-zA-Z0-9+/$%~_-|!.]*)><>(processingTime|sessionCounter|rejectedSessions|expiredSessions):'
  name: tomcat_session_$3_total
  labels:
    context: "$2"
    host: "$1"
  help: Tomcat session $3 total
  type: COUNTER
- pattern: 'org.mongodb.driver.*'
- pattern: 'com.ubnt.*'

prometheus.yml

---
- job_name: cloudkey_jmx
    static_configs:
      - targets: ['ck-plus:9560']

  - job_name: cloudkey_telegraf
    static_configs:
      - targets: ['ck-plus:9273']

setup.sh

curl -sL https://repos.influxdata.com/influxdb.key | sudo apt-key add -
sudo tee /etc/apt/sources.list.d/influx.list << EOT
deb https://repos.influxdata.com/debian stretch stable
EOT
sudo apt-get update
sudo apt-get install -y telegraf ntp apcupsd
grep -vE '^\s*(#|$)' /etc/telegraf/telegraf.conf.sample | grep -v 'influxdb' | sudo tee /etc/telegraf/telegraf.conf
echo '[[inputs.net]]' | sudo tee /etc/telegraf/telegraf.d/net.conf
echo '[[inputs.ntpq]]' | sudo tee /etc/telegraf/telegraf.d/ntp.conf
echo '[[inputs.mongodb]]
  servers = ["mongodb://127.0.0.1:27117"]
  gather_perdb_stats = true
  gather_col_stats = true
  gather_top_stat = true' | sudo tee /etc/telegraf/telegraf.d/mongodb.conf
sudo tee /etc/telegraf/telegraf.d/prometheus.conf << EOT
[[outputs.prometheus_client]]
  # Address to listen on
  listen = ":9273"
EOT
sudo service telegraf restart

mkdir /opt/jmx_exporter
curl -q -L https://search.maven.org/remotecontent?filepath=io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar -o /opt/jmx_exporter/jmx_prometheus_javaagent.jar
vim /opt/jmx_exporter/jmx-config.yml
# /lib/systemd/system/unifi.service.d/unifi.conf overrides the full ExecStart in the Cloudkey firmware
# and removes the JVM_EXTRA_OPTS environment variable, so we have to override that in an ugly way.
cat > /lib/systemd/system/unifi.service.d/unifi.jmx.conf << EOT
[Service]
Environment="JVM_EXTRA_OPTS=-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent.jar=9560:/opt/jmx_exporter/jmx-config.yml"
ExecStart=
ExecStart=/usr/bin/java -javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent.jar=9560:/opt/jmx_exporter/jmx-config.yml -Dfile.encoding=UTF-8 -Djava.awt.headless=true -Dapple.awt.UIElement=true -Dunifi.core.enabled=true -Dorg.xerial.snappy.tempdir=/usr/lib/unifi/run -Xmx768M -Xss1024K -XX:MinHeapFreeRatio=0 -XX:MaxHeapFreeRatio=50 -XX:-TieredCompilation -XX:+ExitOnOutOfMemoryError -XX:+CrashOnOutOfMemoryError -XX:ErrorFile=/usr/lib/unifi/logs/hs_err_pid%p.log -XX:+UnlockExperimentalVMOptions -XX:-UseContainerSupport -jar /usr/lib/unifi/lib/ace.jar start
EOT
systemctl daemon-reload
service unifi restart