quick script to compile a list of nodes that are failing plans on nomad clients

check.py

import requests
import json
import os

url = "https://grafana.rbx.com"

headers = {
    "Authorization": "Bearer " + os.environ["GRAFANA_TOKEN"],
    "Accept": "application/json",
    "Content-Type": "application/json",
}


def api_call(method, endpoint, params={}, data={}):
    global url
    if url[-1] == "/":
        url = url[:-1]
    if endpoint[0] == "/":
        endpoint = endpoint[1:]
    return requests.request(
        method,
        f"{url}/{endpoint}",
        params=params,
        data=json.dumps(data),
        headers=headers,
    ).json()


def get_datasource(ds):
    return api_call("GET", "api/datasources/name/" + ds)


def query(ds_name, query):
    datasource = get_datasource(ds_name)
    return api_call(
        "POST",
        "api/ds/query",
        data={
            "queries": [
                {
                    "expr": query,
                    "datasource": {
                        "uid": datasource["uid"],
                    },
                },
            ],
            "from": "now-4h",
            "to": "now",
        },
    )


def nomad_node_status(node_id):
    return requests.get(
        f"{os.environ['NOMAD_ADDR']}/v1/node/{node_id}",
        headers={"X-Nomad-Token": os.environ["NOMAD_TOKEN"]},
    ).json()


if __name__ == "__main__":
    q = "group (nomad_nomad_plan_node_rejected{}[5m]) by (node_id)"
    data = query("core-infra-mon-sitetest3", q)
    inventory = {"all": {"hosts": []}, "_meta": {"hostvars": {}}}
    node_ids = []
    for frame in data["results"]["A"]["frames"]:
        for field in frame["schema"]["fields"]:
            if (
                "labels" in field
                and "node_id" in field["labels"]
                and field["labels"]["node_id"] not in node_ids
            ):
                node_ids.append(field["labels"]["node_id"])

    for node_id in node_ids:
        status = nomad_node_status(node_id)
        inventory["all"]["hosts"].append(
            {
                status["Attributes"]["unique.network.ip-address"]: {
                    "nomad_id": node_id,
                }
            }
        )
    print(json.dumps(inventory, sort_keys=False, indent=2))