Moloch ECS Conversion Pipeline

moloch-ecs

PUT _ingest/pipeline/moloch-ecs
{
  "description": "Ingest Moloch indices in ECS format.",
  "processors": [
    {
      "rename": {
        "field": "dstIp",
        "target_field": "destination.ip",
        "description": "DESTINATION IP",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dstBytes",
        "target_field": "destination.bytes",
        "description": "DESTINATION BYTES",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dstDataBytes",
        "target_field": "server.bytes",
        "description": "SERVER BYTES",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dstMac",
        "target_field": "destination.mac",
        "description": "DESTINATION MAC",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dstPort",
        "target_field": "destination.port",
        "description": "DESTINATION PORT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dstPackets",
        "target_field": "destination.packets",
        "description": "DESTINATION PACKETS",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dstRIR",
        "target_field": "destination.geo.registry",
        "description": "DESTINATION GEO REGISTRY",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dstOui",
        "target_field": "destination.mac_oui",
        "description": "DESTINATION MAC OUI",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dstOuiCnt",
        "target_field": "destination.mac_oui_count",
        "description": "DESTINATION MAC OUI COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dstMacCnt",
        "target_field": "destination.mac_count",
        "description": "DESTINATION MAC COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "srcRIR",
        "target_field": "source.geo.registry",
        "description": "SOURCE GEO REGISTRY",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "srcIp",
        "target_field": "source.ip",
        "description": "SOURCE IP",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "srcMac",
        "target_field": "source.mac",
        "description": "SOURCE MAC",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "srcMacCnt",
        "target_field": "source.mac_count",
        "description": "SOURCE MAC COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "srcPackets",
        "target_field": "source.packets",
        "description": "SOURCE PACKETS",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "srcPort",
        "target_field": "source.port",
        "description": "SOURCE PORT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "srcBytes",
        "target_field": "source.bytes",
        "description": "SOURCE BYTES",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "srcDataBytes",
        "target_field": "client.bytes",
        "description": "SOURCE DATA BYTES as CLIENT BYTES",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "srcOui",
        "target_field": "source.mac_oui",
        "description": "SOURCE MAC OUI",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "srcOuiCnt",
        "target_field": "source.mac_oui_count",
        "description": "SOURCE MAC OUI COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.useragent",
        "target_field": "user_agent.original",
        "description": "USER AGENT FULL ORIGINAL",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.host",
        "target_field": "url.domain",
        "description": "URL DOMAIN OF REQUEST",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.uri",
        "target_field": "url.full",
        "description": "FULL URL DOMAIN OF REQUEST",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.method",
        "target_field": "http.request.method",
        "description": "HTTP REQUEST METHOD",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.methodCnt",
        "target_field": "http.request.method_count",
        "description": "HTTP REQUEST METHOD COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.pathCnt",
        "target_field": "url.path_count",
        "description": "HTTP PATH COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.path",
        "target_field": "url.path",
        "description": "HTTP PATH",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.requestHeaderCnt",
        "target_field": "http.request.header_count",
        "description": "HTTP REQUEST HEADER COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.requestHeader",
        "target_field": "http.request.header",
        "description": "HTTP REQUEST HEADER",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.responseHeader",
        "target_field": "http.response.header",
        "description": "HTTP RESPONSE HEADER",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.responseHeaderCnt",
        "target_field": "http.response.header_count",
        "description": "HTTP RESPONSE HEADER COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.response-content-type",
        "target_field": "http.response.content_type",
        "description": "HTTP RESPONSE TYPE",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.response-content-typeCnt",
        "target_field": "http.response.content_type_count",
        "description": "HTTP RESPONSE TYPE COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.statuscode",
        "target_field": "http.response.status_code",
        "description": "HTTP RESPONSE STATUS CODE",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.statuscodeCnt",
        "target_field": "http.response.status_code_count",
        "description": "HTTP RESPONSE STATUS CODE COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.bodyMagic",
        "target_field": "http.response.body.content_type",
        "description": "HTTP RESPONSE BODY CONTENT TYPE",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "http.bodyMagicCnt",
        "target_field": "http.response.body.content_type_count",
        "description": "HTTP RESPONSE BODY CONTENT TYPE COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dns.opcode",
        "target_field": "dns.op_code",
        "description": "DNS OP CODE",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dns.qc",
        "target_field": "dns.question.class",
        "description": "DNS QUESTION CLASS",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dns.qcCnt",
        "target_field": "dns.question.class_count",
        "description": "DNS QUESTION CLASS",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dns.qt",
        "target_field": "dns.question.type",
        "description": "DNS QUESTION TYPE",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dns.qtCnt",
        "target_field": "dns.question.type_count",
        "description": "DNS QUESTION TYPE COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dns.RIR",
        "target_field": "dns.geo.registry",
        "description": "DNS GEO REGISTRY",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dns.status",
        "target_field": "dns.response_code",
        "description": "DNS RESPONSE CODE",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dns.statusCnt",
        "target_field": "dns.response_code_count",
        "description": "DNS RESPONSE CODE COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "dns.ip",
        "target_field": "dns.resolved_ip",
        "description": "DNS RESOLVED IPs",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "cert.notAfter",
        "target_field": "x509.not_after",
        "description": "x509 CERT NOT AFTER",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "cert.notBfter",
        "target_field": "x509.not_before",
        "description": "x509 CERT NOT AFTER",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "protocol",
        "target_field": "network.protocol",
        "description": "PROTOCOL",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "protocolCnt",
        "target_field": "network.protocol_count",
        "description": "PROTOCOL COUNT",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "tls.ja3",
        "target_field": "tls.client.ja3",
        "description": "TLS CLIENT JA3",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "geoip": {
        "field": "source.ip",
        "target_field": "source.geo",
        "ignore_failure": true,
        "description": "GEO IP - SRC",
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "geoip": {
        "field": "destination.ip",
        "target_field": "destination.geo",
        "ignore_failure": true,
        "description": "GEO IP - DST",
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "firstPacket",
        "target_field": "event.start",
        "description": "FIRST PACKET - EVENT START",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "date": {
        "field": "event.start",
        "formats": [
          "UNIX",
          "basic_date_time"
        ],
        "target_field": "event.start",
        "ignore_failure": true,
        "description": "EVENT START TIMESTAMP",
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "lastPacket",
        "target_field": "event.end",
        "description": "LAST PACKET - EVENT END",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "date": {
        "field": "event.end",
        "formats": [
          "UNIX",
          "basic_date_time"
        ],
        "target_field": "event.end",
        "ignore_failure": true,
        "description": "EVENT END TIMESTAMP",
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "ipProtocol",
        "target_field": "network.iana_number",
        "description": "NETWORK IANA NUMBER",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "cert.notAfter",
        "target_field": "x509.not_after",
        "description": "CERT NOT AFTER",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "cert.notBefore",
        "target_field": "x509.not_before",
        "description": "CERT NOT BEFORE",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "rename": {
        "field": "totPackets",
        "target_field": "network.packets",
        "description": "TOTAL PACKETS",
        "ignore_missing": true,
        "on_failure": [
          {
            "set": {
              "value": "{{ _ingest.on_failure_message }}",
              "field": "error.message",
              "description": "Set Error Message Dynamically"
            }
          }
        ]
      }
    },
    {
      "date": {
        "field": "timestamp",
        "formats": [
          "ISO8601",
          "UNIX",
          "UNIX_MS"
        ],
        "ignore_failure": true
      }
    }
  ]
}

Updated to include total packets and cert before|after.