logs-ti_tor.node_activity Ingest Pipeline

logs-ti_tor.node_activity.txt

PUT _ingest/pipeline/logs-ti_tor.node_activity
{
  "description": "Pipeline for parsing TOR node activity. Requires the geoip plugin.",
  "processors": [
    {
      "geoip": {
        "ignore_failure": true,
        "field": "tor.exit.ip",
        "target_field": "tor.exit.geo"
      }
    },
    {
      "geoip": {
        "database_file": "GeoLite2-ASN.mmdb",
        "ignore_failure": true,
        "field": "tor.exit.ip",
        "target_field": "tor.exit.as",
        "if": "ctx.tor?.geo?.country_name != '' || ctx.tor?.geo?.country_name != null "
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.geo.location",
        "value": "{{tor.geo.location.lat}}, {{tor.geo.location.lon}}",
        "if": "  def lat = ctx?.threat?.indicator?.geo?.location?.lat;\r\n  return lat != null && (lat instanceof float || lat instanceof double);\r\n"
      }
    },
    {
      "remove": {
        "field": "tor.exit.as.ip",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "ignore_failure": true,
        "ignore_missing": true,
        "field": "tor.exit.as.asn",
        "target_field": "tor.exit.as.number"
      }
    },
    {
      "rename": {
        "ignore_failure": true,
        "ignore_missing": true,
        "field": "tor.exit.as.organization_name",
        "target_field": "tor.exit.as.organization.name"
      }
    },
    {
      "remove": {
        "ignore_failure": true,
        "ignore_missing": true,
        "field": [
          "event.original",
          "message"
        ]
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "ignore_empty_value": true,
        "field": "tor.routing.ip",
        "value": "{{tor.routing.addresses}}"
      }
    },
    {
      "grok": {
        "ignore_failure": true,
        "ignore_missing": true,
        "field": "tor.routing.ip",
        "patterns": [
          "\\{0=%{IPV4:tor.routing.ipv4.ip}\\:%{INT:tor.routing.ipv4.port}\\, 1=\\[%{IPV6:tor.routing.ipv6.ip}\\]\\:%{INT:tor.routing.ipv6.port}",
          "\\{0=%{IPV4:tor.routing.ipv4.ip}\\:%{INT:tor.routing.ipv4.port}"
        ]
      }
    },
    {
      "geoip": {
        "ignore_failure": true,
        "field": "tor.routing.ipv4.ip",
        "target_field": "tor.routing.ipv4.geo"
      }
    },
    {
      "geoip": {
        "ignore_failure": true,
        "field": "tor.routing.ipv6.ip",
        "target_field": "tor.routing.ipv6.geo"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.routing.ipv4.geo.location",
        "value": "{{tor.routing.ipv4.geo.location.lat}}, {{tor.routing.ipv4.geo.location.lon}}",
        "if": "  def lat = ctx?.tor?.routing?.ipv4?.geo?.location?.lat;\r\n  return lat != null && (lat instanceof float || lat instanceof double);\r\n"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.routing.ipv6.geo.location",
        "value": "{{tor.routing.ipv6.geo.location.lat}}, {{tor.routing.ipv6.geo.location.lon}}",
        "if": "  def lat = ctx?.tor?.routing?.ipv6?.geo?.location?.lat;\r\n  return lat != null && (lat instanceof float || lat instanceof double);\r\n"
      }
    },
    {
      "remove": {
        "ignore_failure": true,
        "ignore_missing": true,
        "field": [
          "tor.routing.ip",
          "tor.routing.ipv6.geo.location.lat",
          "tor.routing.ipv6.geo.location.lon",
          "tor.routing.ipv4.geo.location.lat",
          "tor.routing.ipv4.geo.location.lon",
          "tor.routing.addresses"
        ]
      }
    },
    {
      "fingerprint": {
        "ignore_missing": true,
        "description": "Fingerprints: tor.description, tor.exit.ip, tor.first_seen, tor.last_changed, tor.last_restarted, tor.last_seen, tor.name, tor.routing.addresses to look for any changes and writes to _id",
        "fields": [
          "tor.description",
          "tor.exit.ip",
          "tor.name",
          "tor.routing.addresses",
          "tor.bandwidth.advertised",
          "tor.bandwidth.burst",
          "tor.bandwidth.observed",
          "tor.bandwidth.rate",
          "tor.contact.original",
          "tor.fingerprint",
          "tor.platform"
        ],
        "target_field": "_id",
        "method": "MD5"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.detail.Authority",
        "value": "Relay is a directory authority.",
        "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Authority')"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.detail.BadExit",
        "value": "Exit node that is not recommended (e.g., due to abuse reports or misconfiguration).",
        "if": "ctx.tor?.description != null && ctx.tor?.description.contains('BadExit')"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.detail.Exit",
        "value": "Relay allows traffic to exit the Tor network.",
        "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Exit')"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.detail.Fast",
        "value": "Relay has high bandwidth, based on observed performance.",
        "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Fast')"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.detail.Guard",
        "value": "Relay is suitable for use as an entry node.",
        "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Guard')"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.detail.HSDir",
        "value": "Relay stores hidden service descriptors (used in onion services).",
        "if": "ctx.tor?.description != null && ctx.tor?.description.contains('HSDir')"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.detail.NoEdConsensus",
        "value": "Relay was excluded from the consensus for some reason (e.g., misbehavior).",
        "if": "ctx.tor?.description != null && ctx.tor?.description.contains('NoEdConsensus')"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.detail.Running",
        "value": "Relay is currently running. (Also appears as a top-level filter like in your URL.)",
        "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Running')"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.detail.Stable",
        "value": "Relay has been up for a long time consistently.",
        "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Stable')"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.detail.Valid",
        "value": "Relay is considered \"valid\" by directory authorities.",
        "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Valid')"
      }
    },
    {
      "set": {
        "ignore_failure": true,
        "field": "tor.detail.Unnamed",
        "value": "Relay does not have a configured nickname.",
        "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Unnamed')"
      }
    },
    {
      "set": {
        "field": "threat.indicator.as.network",
        "ignore_failure": true,
        "copy_from": "tor.exit.as.network"
      }
    },
    {
      "set": {
        "field": "threat.indicator.as.number",
        "copy_from": "tor.exit.as.number",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.as.organization.name",
        "copy_from": "tor.exit.as.organization.name",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.first_seen",
        "copy_from": "tor.first_seen",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.geo.city_name",
        "copy_from": "tor.exit.geo.city_name",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.geo.continent_name",
        "copy_from": "tor.exit.geo.continent_name",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.geo.country_iso_code",
        "copy_from": "tor.exit.geo.country_iso_code",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.geo.country_name",
        "copy_from": "tor.exit.geo.country_name",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.geo.location",
        "copy_from": "tor.exit.geo.location",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.geo.region_iso_code",
        "copy_from": "tor.exit.geo.region_iso_code",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.geo.region_name",
        "copy_from": "tor.exit.geo.region_name",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.last_changed",
        "copy_from": "tor.last_changed",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.last_restarted",
        "copy_from": "tor.last_restarted",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "threat.indicator.last_seen",
        "copy_from": "tor.last_seen",
        "ignore_failure": true
      }
    }
  ],
  "on_failure": [
    {
      "set": {
        "field": "error.message",
        "value": "{{ _ingest.on_failure_message }}"
      }
    }
  ]
}