NodeRED flow to accept Nginx proxy access logs via syslog and post to ElasticSearch

nodered-syslog-to-elasticsearch.json

[
    {
        "id": "1dffb66d.88ae0a",
        "type": "function",
        "z": "7be8537a.9bbd5c",
        "name": "Reset dropped entry count",
        "func": "flow.set(\"dropped_entries\", 0);\nreturn null;",
        "outputs": "0",
        "noerr": 0,
        "x": 603.8958129882812,
        "y": 382,
        "wires": []
    },
    {
        "id": "a91a0999.4e93e8",
        "type": "udp in",
        "z": "7be8537a.9bbd5c",
        "name": "syslog",
        "iface": "",
        "port": "1514",
        "ipv": "udp4",
        "multicast": "false",
        "group": "",
        "datatype": "utf8",
        "x": 153.8958282470703,
        "y": 250,
        "wires": [
            [
                "e36b4490.acd448"
            ]
        ]
    },
    {
        "id": "e36b4490.acd448",
        "type": "function",
        "z": "7be8537a.9bbd5c",
        "name": "Strip syslog header",
        "func": "var rx = new RegExp(/^.*nginx:\\ (.*)$/);\n\nmsg.payload = msg.payload.match(rx)[1];\nreturn msg;",
        "outputs": 1,
        "noerr": 0,
        "x": 382.8958282470703,
        "y": 248,
        "wires": [
            [
                "2f0df1f3.e40b3e"
            ]
        ]
    },
    {
        "id": "2f0df1f3.e40b3e",
        "type": "json",
        "z": "7be8537a.9bbd5c",
        "name": "",
        "x": 601.8958282470703,
        "y": 249,
        "wires": [
            [
                "7d718e66.93923"
            ]
        ]
    },
    {
        "id": "b8d024d9.0685d8",
        "type": "catch",
        "z": "7be8537a.9bbd5c",
        "name": "",
        "scope": null,
        "x": 488.89581298828125,
        "y": 48,
        "wires": [
            [
                "a6e2b4be.3d2648"
            ]
        ]
    },
    {
        "id": "a6e2b4be.3d2648",
        "type": "debug",
        "z": "7be8537a.9bbd5c",
        "name": "",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "error",
        "x": 659.8958129882812,
        "y": 57,
        "wires": []
    },
    {
        "id": "73e0d918.73ad68",
        "type": "comment",
        "z": "7be8537a.9bbd5c",
        "name": "Nginx logs to ElasticSearch",
        "info": "Reads access logs from nginx proxies via syslog port, parses and forwards the entry to ElasticSearch for further reporting.",
        "x": 200.8958282470703,
        "y": 67,
        "wires": []
    },
    {
        "id": "7d718e66.93923",
        "type": "http request",
        "z": "7be8537a.9bbd5c",
        "name": "Post to ElasticSearch index",
        "method": "POST",
        "ret": "obj",
        "url": "http://elasticsearch:9200/proxylogs/entry/",
        "tls": "",
        "x": 399.89581298828125,
        "y": 326,
        "wires": [
            [
                "adf6fc8f.f89f4",
                "f3e67d3a.60c6e"
            ]
        ]
    },
    {
        "id": "f3e67d3a.60c6e",
        "type": "debug",
        "z": "7be8537a.9bbd5c",
        "name": "",
        "active": false,
        "console": "false",
        "complete": "false",
        "x": 650.8958129882812,
        "y": 323,
        "wires": []
    },
    {
        "id": "adf6fc8f.f89f4",
        "type": "switch",
        "z": "7be8537a.9bbd5c",
        "name": "Check result",
        "property": "payload.result",
        "propertyType": "msg",
        "rules": [
            {
                "t": "eq",
                "v": "created",
                "vt": "str"
            },
            {
                "t": "else"
            }
        ],
        "checkall": "true",
        "outputs": 2,
        "x": 347.89581298828125,
        "y": 396,
        "wires": [
            [
                "1dffb66d.88ae0a"
            ],
            [
                "33330424.7fcedc"
            ]
        ]
    },
    {
        "id": "200a1a2b.2e47c6",
        "type": "http in",
        "z": "7be8537a.9bbd5c",
        "name": "",
        "url": "/web-proxy-logs/metrics",
        "method": "get",
        "swaggerDoc": "",
        "x": 222.8958282470703,
        "y": 499,
        "wires": [
            [
                "c3de9938.ffcb58"
            ]
        ]
    },
    {
        "id": "7f770b5a.d1e4b4",
        "type": "http response",
        "z": "7be8537a.9bbd5c",
        "name": "",
        "x": 689.8958282470703,
        "y": 497,
        "wires": []
    },
    {
        "id": "c3de9938.ffcb58",
        "type": "template",
        "z": "7be8537a.9bbd5c",
        "name": "Prometheus metrics",
        "field": "payload",
        "fieldType": "msg",
        "format": "handlebars",
        "syntax": "mustache",
        "template": "# HELP web_proxy_dropped_entries Number of entries dropped by ElasticSearch recently\n# TYPE web_proxy_dropped_entries gauge\nweb_proxy_dropped_entries{instance=\"{{flow.environment}}\",job=\"webproxy\"} {{flow.dropped_entries}}",
        "x": 483.8958282470703,
        "y": 499,
        "wires": [
            [
                "7f770b5a.d1e4b4"
            ]
        ]
    },
    {
        "id": "560d29dc.148588",
        "type": "inject",
        "z": "7be8537a.9bbd5c",
        "name": "Initialisation",
        "topic": "",
        "payload": "",
        "payloadType": "str",
        "repeat": "",
        "crontab": "",
        "once": true,
        "x": 182.8958282470703,
        "y": 141,
        "wires": [
            [
                "8d41d86f.0ab0e8"
            ]
        ]
    },
    {
        "id": "33330424.7fcedc",
        "type": "function",
        "z": "7be8537a.9bbd5c",
        "name": "Increment dropped entry count",
        "func": "var count = flow.get(\"dropped_entries\");\nflow.set(\"dropped_entries\", count + 1);\nreturn null;",
        "outputs": "0",
        "noerr": 0,
        "x": 616.8958129882812,
        "y": 433,
        "wires": []
    },
    {
        "id": "8d41d86f.0ab0e8",
        "type": "change",
        "z": "7be8537a.9bbd5c",
        "name": "Initialise flow variables",
        "rules": [
            {
                "t": "set",
                "p": "environment",
                "pt": "flow",
                "to": "production",
                "tot": "str"
            },
            {
                "t": "set",
                "p": "dropped_entries",
                "pt": "msg",
                "to": "0",
                "tot": "num"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 389,
        "y": 142,
        "wires": [
            []
        ]
    }
]