NiFi: Better alert checks for Ambari

README.md

Ambari Alert to check if NiFi's /nifi-api/flow/process-groups/root is working

Installation:

• Save alert_check_nifi_processgroup.py to /var/lib/ambari-server/resources/common-services/NIFI/1.0.0/packages/scripts/
• sudo chmod a+rx alert_check_nifi_processgroup.py
• Save alerts.json.patch to /tmp/
• Apply patch: sudo patch -b /var/lib/ambari-server/resources/common-services/NIFI/1.0.0/alerts.json /tmp/alerts.json.patch
• On Ambari Server: sudo ambari-server refresh-stack-hash
• On all hosts: sudo ambari-agent restart

alert_check_nifi_processgroup.py

"""Check that `GET /nifi-api/flow/process-groups/root`"""
import socket
import traceback

from resource_management.core.exceptions import ComponentIsNotRunning
from resource_management.libraries.functions import get_kinit_path
from resource_management.core.resources import Execute

NIFI_PORT_KEY = '{{nifi-ambari-config/nifi.node.port}}'
NIFI_PORT_SSL_KEY = '{{nifi-ambari-config/nifi.node.ssl.port}}'

SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
NIFI_KEYTAB_KEY = '{{nifi-properties/nifi.kerberos.service.keytab.location}}'
NIFI_PRINCIPAL_KEY = '{{nifi-properties/nifi.kerberos.service.principal}}'
NIFI_USER_KEY = '{{nifi-env/nifi_user}}'

UI_SSL_ENABLED = '{{nifi-ambari-ssl-config/nifi.node.ssl.isenabled}}'

KRB_EXEC_SEARCH_PATHS_KEY = '{{kerberos-env/executable_search_paths}}'

RESULT_CODE_OK = 'OK'
RESULT_CODE_CRITICAL = 'CRITICAL'
RESULT_CODE_UNKNOWN = 'UNKNOWN'


def get_tokens():
    return (NIFI_USER_KEY, UI_SSL_ENABLED, SECURITY_ENABLED_KEY,
            NIFI_KEYTAB_KEY, NIFI_PRINCIPAL_KEY, KRB_EXEC_SEARCH_PATHS_KEY,
            NIFI_PORT_KEY, NIFI_PORT_SSL_KEY)


def security_auth(configs, host_name, nifi_user):
    """
  Call kinit before pursuit with any other action

  :type configs dict
  :type host_name str
  :type nifi_user str
  """
    nifi_kerberos_keytab = configs[
        NIFI_KEYTAB_KEY] if NIFI_KEYTAB_KEY in configs else None
    nifi_principal = configs[NIFI_PRINCIPAL_KEY].replace(
        '_HOST', host_name.lower()) if NIFI_PRINCIPAL_KEY in configs else None
    krb_executable_search_paths = configs[
        KRB_EXEC_SEARCH_PATHS_KEY] if KRB_EXEC_SEARCH_PATHS_KEY in configs else None
    kinit_path_local = get_kinit_path(krb_executable_search_paths)

    if not nifi_principal or not nifi_kerberos_keytab:
        raise ComponentIsNotRunning(
            "Error: nifi principal or nifi kerberos keytab can't be None")

    kinit_cmd = "{kinit_path_local} -kt {nifi_kerberos_keytab} {nifi_principal};".format(
        kinit_path_local=kinit_path_local,
        nifi_kerberos_keytab=nifi_kerberos_keytab,
        nifi_principal=nifi_principal)

    Execute(kinit_cmd, user=nifi_user)


def execute(configs, parameters=None, host_name=None):

    if configs is None:
        return 'UNKNOWN', [
            'There were no configurations supplied to the script.'
        ]

    if host_name is None:
        host_name = socket.getfqdn()

    nifi_user = configs[NIFI_USER_KEY]
    ui_ssl_enabled = str(configs[UI_SSL_ENABLED]).upper(
    ) == 'TRUE' if UI_SSL_ENABLED in configs else False
    nifi_port = configs[
        NIFI_PORT_SSL_KEY] if UI_SSL_ENABLED in configs else configs[
            NIFI_PORT_KEY]
    security_enabled = str(configs[SECURITY_ENABLED_KEY]).upper(
    ) == 'TRUE' if SECURITY_ENABLED_KEY in configs else False

    try:
        if security_enabled:
            security_auth(configs, host_name, nifi_user)
    except Exception as e:
        return RESULT_CODE_CRITICAL, ["kinit error: " + str(e)]

    curl_auth = None
    if ui_ssl_enabled:
        scheme = "https"
        curl_auth = "--cert /etc/security/certificates/host.crt --key /etc/security/certificates/host.key"
    else:
        scheme = "http"

    cmd = "curl -k -m 5 -sS -w' HTTPSTATUS:%{{http_code}}' {curl_auth} {scheme}://{host_name}:{nifi_port}/nifi-api/flow/process-groups/root | sed '/processGroupFlow.* HTTPSTATUS:200$/!{{$q1}}'".format(
        scheme=scheme,
        curl_auth=curl_auth,
        host_name=host_name,
        nifi_port=nifi_port)

    try:
        Execute(cmd, tries=2, try_sleep=3, user=nifi_user, logoutput=True)
    except ComponentIsNotRunning as ex:
        return RESULT_CODE_CRITICAL, [str(ex)]
    except:
        return RESULT_CODE_CRITICAL, [
            "CRITICAL: {0}".format(traceback.format_exc())
        ]

    return RESULT_CODE_OK, ["OK: ({0})".format(cmd)]

alert_nifi_rootprocessgroup.sh

#!/usr/bin/env bash

## this file is only for testing

NO_PROXY='*'

url="https://$(hostname -f):9091/nifi-api/flow/process-groups/root"
code=200

response="$(curl -k -m 5 -sS -w 'HTTPSTATUS:%{http_code}' --key /etc/security/certificates/host.key --cert /etc/security/certificates/host.crt ${url} )"
body="$(echo ${response} | sed -E 's/HTTPSTATUS\:[0-9]{3}$//')"
status="$(echo ${response} | tr -d '\n' | sed -E 's/.*HTTPSTATUS:([0-9]{3})$/\1/')"

if test ${status} -ne ${code}; then
    echo "Fail: HTTP ${status}"
    echo "${body}"
fi

#echo ${response} | sed '/HTTPSTATUS:200$/!{{$q1}}'

alerts.json.patch

--- /var/lib/ambari-server/resources/common-services/NIFI/1.0.0/alerts.json.orig	2019-04-05 11:28:57.000000000 +0100
+++ /var/lib/ambari-server/resources/common-services/NIFI/1.0.0/alerts.json	2019-04-05 11:28:55.000000000 +0100
@@ -12,6 +12,17 @@
           "type": "SCRIPT",
           "path": "NIFI/1.0.0/package/scripts/alert_check_nifi.py"
         }
+      },
+      {
+        "name": "nifi_processgroup_status",
+        "label": "NiFi Process Group Status",
+        "description": "This host-level alert is triggered if /nifi-api/flow/process-groups/root is not available.",
+        "interval": 1,
+        "scope": "ANY",
+        "source": {
+          "type": "SCRIPT",
+          "path": "NIFI/1.0.0/package/scripts/alert_check_nifi_processgroup.py"
+        }
       }
     ]
   }