indigo423

## Overview

This article describes how OpenNMS is configured to collect and store metrics from an external Kubernetes cluster using Prometheus as the data source. It covers what is collected, how the collection pipeline is structured, and how data is persisted in RRD storage for threshold evaluation.

## Collection Architecture

The Kubernetes cluster exposes Prometheus metrics on port 9090. OpenNMS Collectd connects to that endpoint and issues PromQL queries on a 30-second polling interval. Results are written as JRB files into the OpenNMS RRD store. Threshold definitions then evaluate those files continuously and generate events when values cross defined limits.

Collectd -> Prometheus (:9090) -> PromQL -> JRB (RRD) -> Threshold Engine -> Events

indigo423

#/usr/bin/env bash

VERSION="2c"
COMMUNITY="s3cur17y"
TARGET="192.168.11.77"
PORT="10162"

iptables -t nat -A POSTROUTING -d $TARGET -p udp --dport $PORT -j SNAT --to 10.42.255.1
snmptrap -v ${VERSION} -c ${COMMUNITY} ${TARGET}:${PORT} "" \
    .1.3.6.1.4.1.61509.1.999 \

indigo423

datacollection-config xmlns="http://xmlns.opennms.org/xsd/config/datacollection" rrdRepository="/var/lib/opennms/rrd/snmp/">
   <snmp-collection name="default" snmpStorageFlag="select">
      <rrd step="300">
         <rra>RRA:AVERAGE:0.5:1:2016</rra>
         <rra>RRA:AVERAGE:0.5:12:1488</rra>
         <rra>RRA:AVERAGE:0.5:288:366</rra>
         <rra>RRA:MAX:0.5:288:366</rra>
         <rra>RRA:MIN:0.5:288:366</rra>
      </rrd>
      <!-- Custom collection groups for the default collection go here -->

indigo423

<?xml version="1.0" encoding="UTF-8"?>
<!-- WARN here is just for internal log4j messages and does not effect logging in general -->
<configuration status="WARN" monitorInterval="60">

  <!--
      The logging in OpenNMS is designed to associate a 'prefix' with each thread that runs.
      This prefix is used to indicate what 'subsystem' the thread is doing work for so that
      logging for the subsystem can all be directed to the same log.  The way this needs to work 
      in the code is that when a thread is doing work for subsystem (ie when it starts a method for
      this system) it sets its prefix in a ThreadLocal way.  All logs on this thread will then

indigo423

<events xmlns="http://xmlns.opennms.org/xsd/eventconf">
   <global>
      <security>
         <doNotOverride>logmsg</doNotOverride>
         <doNotOverride>operaction</doNotOverride>
         <doNotOverride>autoaction</doNotOverride>
         <doNotOverride>tticket</doNotOverride>
         <doNotOverride>script</doNotOverride>
      </security>
   </global>

indigo423

<?xml version="1.0"?>
<syslogd-configuration>
    <!--
      optional attribute "timezone":
          - if set it will be used as default timezone if no timezone is given
          - if not set the system time zone will be used
    -->
    <configuration
            syslog-port="10514"
            new-suspect-on-message="false"

indigo423

<events xmlns="http://xmlns.opennms.org/xsd/eventconf">
    <event>
        <uei>uei.opennms.org/hpe/demo/card/raise</uei>
        <event-label>HPE demo event: Card raise event</event-label>
        <descr>%parm[all]%</descr>
        <logmsg dest="logndisplay">%parm[message]%</logmsg>
        <severity>Warning</severity>
        <alarm-data reduction-key="%uei%:%hostname%:%parm[card]%"
                    alarm-type="1"
                    auto-clean="false"/>

indigo423

<?xml version="1.0"?>
<syslogd-configuration-group>
    <ueiList>
        <ueiMatch>
            <process-match expression="^tester$" />
            <match type="regex" expression="^error:origin=(.*?):system=(.*?):severity=(.*?):message=(.*?):card=(.*?)$" />
            <uei>uei.opennms.org/hpe/demo/card/raise</uei>
            <parameter-assignment matching-group="1" parameter-name="origin" />
            <parameter-assignment matching-group="2" parameter-name="system" />
            <parameter-assignment matching-group="3" parameter-name="severity" />

indigo423

<events xmlns="http://xmlns.opennms.org/xsd/eventconf">
   <event>
      <uei>uei.opennms.org/hpe/syslogdemo/raise</uei>
      <event-label>HPE demo event: logger raise event</event-label>
      <descr>&lt;p&gt;The interface %interface% generated a Syslog Message.&lt;br&gt;
                        Node ID: %nodeid%&lt;br&gt;
                        Host: %nodelabel%&lt;br&gt;
                        Interface: %interface% &lt;br&gt;
                        Message: %parm[syslogmessage]% &lt;br&gt;
                        Process: %parm[process]% &lt;br&gt;

indigo423

# HELP jmx_config_reload_failure_total Number of times configuration have failed to be reloaded.
# TYPE jmx_config_reload_failure_total counter
jmx_config_reload_failure_total 0.0
# HELP jmx_config_reload_success_total Number of times configuration have successfully been reloaded.
# TYPE jmx_config_reload_success_total counter
jmx_config_reload_success_total 0.0
# HELP jmx_exporter_build_info JMX Exporter build information
# TYPE jmx_exporter_build_info gauge
jmx_exporter_build_info{name="jmx_prometheus_javaagent",version="1.3.0"} 1
# HELP jmx_scrape_cached_beans Number of beans with their matching rule cached