Last active
June 11, 2025 13:59
-
-
Save iamhowardtheduck/bf06ad3622253de8a90f4a030e02492a to your computer and use it in GitHub Desktop.
logs-ti_tor.node_activity content
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # First create your ingest pipeline to parse the data in DevTools: | |
| PUT _ingest/pipeline/logs-ti_tor.node_activity | |
| { | |
| "description": "Pipeline for parsing TOR node activity. Requires the geoip plugin.", | |
| "processors": [ | |
| { | |
| "geoip": { | |
| "ignore_failure": true, | |
| "field": "tor.exit.ip", | |
| "target_field": "tor.exit.geo", | |
| "description": "Adds geo-based data from the tor.exit.ip field" | |
| } | |
| }, | |
| { | |
| "geoip": { | |
| "database_file": "GeoLite2-ASN.mmdb", | |
| "ignore_failure": true, | |
| "field": "tor.exit.ip", | |
| "target_field": "tor.exit.as", | |
| "description": "Adds geo-based ASN data from the tor.exit.ip field if the first geoip processor created a country_name field.", | |
| "if": "ctx.tor?.geo?.country_name != '' || ctx.tor?.geo?.country_name != null " | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.geo.location", | |
| "value": "{{tor.geo.location.lat}}, {{tor.geo.location.lon}}", | |
| "description": "Creates the field tor.geo.location from the latitude and longitude from our first geo processor if the lat field is not null and formatted like a double.", | |
| "if": " def lat = ctx?.threat?.indicator?.geo?.location?.lat;\r\n return lat != null && (lat instanceof float || lat instanceof double);\r\n" | |
| } | |
| }, | |
| { | |
| "remove": { | |
| "field": "tor.exit.as.ip", | |
| "ignore_missing": true, | |
| "description": "Removes the tor.exit.as.ip field, as this is created from the 2nd geoip processor and is redundant." | |
| } | |
| }, | |
| { | |
| "rename": { | |
| "ignore_failure": true, | |
| "ignore_missing": true, | |
| "field": "tor.exit.as.asn", | |
| "target_field": "tor.exit.as.number", | |
| "description": "Renames the tor.exit.as.asn field to the expected ECS format." | |
| } | |
| }, | |
| { | |
| "rename": { | |
| "ignore_failure": true, | |
| "ignore_missing": true, | |
| "field": "tor.exit.as.organization_name", | |
| "target_field": "tor.exit.as.organization.name", | |
| "description": "Renames the tor.exit.as.organization_name field to the expected ECS format." | |
| } | |
| }, | |
| { | |
| "remove": { | |
| "ignore_failure": true, | |
| "ignore_missing": true, | |
| "description": "Removes the redundant original event from each document to reduce storage footprint.", | |
| "field": [ | |
| "event.original", | |
| "message" | |
| ] | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "ignore_empty_value": true, | |
| "field": "tor.routing.ip", | |
| "value": "{{tor.routing.addresses}}", | |
| "description": "Creates the field tor.routing.ip from tor.routing.addresses to be used in a GROK down the pipe." | |
| } | |
| }, | |
| { | |
| "grok": { | |
| "ignore_failure": true, | |
| "ignore_missing": true, | |
| "field": "tor.routing.ip", | |
| "description": "The tor.routing.ip field contains either just an IPv4 and port, or sometimes both IPv4 & IPv6; this captures both.", | |
| "patterns": [ | |
| "\\{0=%{IPV4:tor.routing.ipv4.ip}\\:%{INT:tor.routing.ipv4.port}\\, 1=\\[%{IPV6:tor.routing.ipv6.ip}\\]\\:%{INT:tor.routing.ipv6.port}", | |
| "\\{0=%{IPV4:tor.routing.ipv4.ip}\\:%{INT:tor.routing.ipv4.port}" | |
| ] | |
| } | |
| }, | |
| { | |
| "geoip": { | |
| "ignore_failure": true, | |
| "field": "tor.routing.ipv4.ip", | |
| "description": "Creates geo-based data from the IPv4 address.", | |
| "target_field": "tor.routing.ipv4.geo" | |
| } | |
| }, | |
| { | |
| "geoip": { | |
| "ignore_failure": true, | |
| "field": "tor.routing.ipv6.ip", | |
| "description": "Creates geo-based data from the IPv6 address.", | |
| "target_field": "tor.routing.ipv6.geo" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.routing.ipv4.geo.location", | |
| "value": "{{tor.routing.ipv4.geo.location.lat}}, {{tor.routing.ipv4.geo.location.lon}}", | |
| "description": "Creates the location field from the IPv4 lat/long if the lat field is not null and formatted like a double.", | |
| "if": " def lat = ctx?.tor?.routing?.ipv4?.geo?.location?.lat;\r\n return lat != null && (lat instanceof float || lat instanceof double);\r\n" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.routing.ipv6.geo.location", | |
| "value": "{{tor.routing.ipv6.geo.location.lat}}, {{tor.routing.ipv6.geo.location.lon}}", | |
| "description": "Creates the location field from the IPv6 lat/long if the lat field is not null and formatted like a double.", | |
| "if": " def lat = ctx?.tor?.routing?.ipv6?.geo?.location?.lat;\r\n return lat != null && (lat instanceof float || lat instanceof double);\r\n" | |
| } | |
| }, | |
| { | |
| "remove": { | |
| "ignore_failure": true, | |
| "ignore_missing": true, | |
| "description": "Removes redundant or unused fields.", | |
| "field": [ | |
| "tor.routing.ip", | |
| "tor.routing.ipv6.geo.location.lat", | |
| "tor.routing.ipv6.geo.location.lon", | |
| "tor.routing.ipv4.geo.location.lat", | |
| "tor.routing.ipv4.geo.location.lon", | |
| "tor.routing.addresses" | |
| ] | |
| } | |
| }, | |
| { | |
| "fingerprint": { | |
| "ignore_missing": true, | |
| "description": "Fingerprints: tor.description, tor.exit.ip, tor.name, tor.routing.addresses to look for any changes and writes to _id. This ensures there are only ever updates to core pieces of information.", | |
| "fields": [ | |
| "tor.description", | |
| "tor.exit.ip", | |
| "tor.name", | |
| "tor.routing.addresses", | |
| "tor.bandwidth.advertised", | |
| "tor.bandwidth.burst", | |
| "tor.bandwidth.observed", | |
| "tor.bandwidth.rate", | |
| "tor.contact.original", | |
| "tor.fingerprint", | |
| "tor.platform" | |
| ], | |
| "target_field": "_id", | |
| "method": "MD5" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.detail.Authority", | |
| "value": "Relay is a directory authority.", | |
| "description": "If this value is present, then create a human readable description of what the function is.", | |
| "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Authority')" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.detail.BadExit", | |
| "value": "Exit node that is not recommended (e.g., due to abuse reports or misconfiguration).", | |
| "description": "If this value is present, then create a human readable description of what the function is.", | |
| "if": "ctx.tor?.description != null && ctx.tor?.description.contains('BadExit')" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.detail.Exit", | |
| "value": "Relay allows traffic to exit the Tor network.", | |
| "description": "If this value is present, then create a human readable description of what the function is.", | |
| "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Exit')" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.detail.Fast", | |
| "value": "Relay has high bandwidth, based on observed performance.", | |
| "description": "If this value is present, then create a human readable description of what the function is.", | |
| "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Fast')" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.detail.Guard", | |
| "value": "Relay is suitable for use as an entry node.", | |
| "description": "If this value is present, then create a human readable description of what the function is.", | |
| "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Guard')" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.detail.HSDir", | |
| "value": "Relay stores hidden service descriptors (used in onion services).", | |
| "description": "If this value is present, then create a human readable description of what the function is.", | |
| "if": "ctx.tor?.description != null && ctx.tor?.description.contains('HSDir')" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.detail.NoEdConsensus", | |
| "value": "Relay was excluded from the consensus for some reason (e.g., misbehavior).", | |
| "description": "If this value is present, then create a human readable description of what the function is.", | |
| "if": "ctx.tor?.description != null && ctx.tor?.description.contains('NoEdConsensus')" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.detail.Running", | |
| "value": "Relay is currently running. (Also appears as a top-level filter like in your URL.)", | |
| "description": "If this value is present, then create a human readable description of what the function is.", | |
| "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Running')" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.detail.Stable", | |
| "value": "Relay has been up for a long time consistently.", | |
| "description": "If this value is present, then create a human readable description of what the function is.", | |
| "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Stable')" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.detail.Valid", | |
| "value": "Relay is considered \"valid\" by directory authorities.", | |
| "description": "If this value is present, then create a human readable description of what the function is.", | |
| "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Valid')" | |
| } | |
| }, | |
| { | |
| "set": { | |
| "ignore_failure": true, | |
| "field": "tor.detail.Unnamed", | |
| "value": "Relay does not have a configured nickname.", | |
| "description": "If this value is present, then create a human readable description of what the function is.", | |
| "if": "ctx.tor?.description != null && ctx.tor?.description.contains('Unnamed')" | |
| } | |
| } | |
| ], | |
| "on_failure": [ | |
| { | |
| "set": { | |
| "field": "error.message", | |
| "value": "{{ _ingest.on_failure_message }}", | |
| "description": "If anything fails, write the error to the error.message field." | |
| } | |
| } | |
| ] | |
| } | |
| # Next, create your index template to map the data while still in DevTools: | |
| PUT _index_template/logs-ti_tor.node_activity | |
| { | |
| "priority": 200, | |
| "template": { | |
| "settings": { | |
| "index": { | |
| "mode": "standard", | |
| "number_of_replicas": "1", | |
| "default_pipeline": "logs-ti_tor.node_activity" | |
| } | |
| }, | |
| "mappings": { | |
| "properties": { | |
| "data_stream": { | |
| "type": "object", | |
| "properties": { | |
| "namespace": { | |
| "type": "keyword" | |
| }, | |
| "type": { | |
| "type": "keyword" | |
| }, | |
| "dataset": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| } | |
| } | |
| }, | |
| "tor": { | |
| "type": "object", | |
| "properties": { | |
| "measured": { | |
| "type": "boolean" | |
| }, | |
| "first_seen": { | |
| "format": "strict_date_optional_time||epoch_millis||yyyy-MM-dd HH:mm:ss", | |
| "index": true, | |
| "ignore_malformed": false, | |
| "store": false, | |
| "type": "date", | |
| "doc_values": true | |
| }, | |
| "last_seen": { | |
| "format": "strict_date_optional_time||epoch_millis||yyyy-MM-dd HH:mm:ss", | |
| "index": true, | |
| "ignore_malformed": false, | |
| "store": false, | |
| "type": "date", | |
| "doc_values": true | |
| }, | |
| "guard": { | |
| "type": "object", | |
| "properties": { | |
| "probability": { | |
| "type": "half_float" | |
| } | |
| } | |
| }, | |
| "description": { | |
| "eager_global_ordinals": false, | |
| "index_phrases": false, | |
| "fielddata": false, | |
| "norms": true, | |
| "index": true, | |
| "store": false, | |
| "type": "text", | |
| "index_options": "positions" | |
| }, | |
| "platform": { | |
| "type": "keyword" | |
| }, | |
| "running": { | |
| "type": "boolean" | |
| }, | |
| "routing": { | |
| "type": "object", | |
| "properties": { | |
| "addresses": { | |
| "type": "text" | |
| }, | |
| "ipv4": { | |
| "type": "object", | |
| "properties": { | |
| "geo": { | |
| "type": "object", | |
| "properties": { | |
| "continent_name": { | |
| "type": "keyword" | |
| }, | |
| "region_iso_code": { | |
| "type": "keyword" | |
| }, | |
| "city_name": { | |
| "type": "keyword" | |
| }, | |
| "country_iso_code": { | |
| "type": "keyword" | |
| }, | |
| "country_name": { | |
| "type": "keyword" | |
| }, | |
| "location": { | |
| "type": "geo_point" | |
| }, | |
| "region_name": { | |
| "type": "keyword" | |
| } | |
| } | |
| }, | |
| "port": { | |
| "type": "long" | |
| }, | |
| "ip": { | |
| "type": "ip" | |
| } | |
| } | |
| }, | |
| "ipv6": { | |
| "type": "object", | |
| "properties": { | |
| "geo": { | |
| "type": "object", | |
| "properties": { | |
| "continent_name": { | |
| "type": "keyword" | |
| }, | |
| "region_iso_code": { | |
| "type": "keyword" | |
| }, | |
| "city_name": { | |
| "type": "keyword" | |
| }, | |
| "country_iso_code": { | |
| "type": "keyword" | |
| }, | |
| "country_name": { | |
| "type": "keyword" | |
| }, | |
| "location": { | |
| "type": "geo_point" | |
| }, | |
| "region_name": { | |
| "type": "keyword" | |
| } | |
| } | |
| }, | |
| "port": { | |
| "type": "long" | |
| }, | |
| "ip": { | |
| "type": "ip" | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "hostname": { | |
| "type": "object", | |
| "properties": { | |
| "verified": { | |
| "type": "keyword" | |
| }, | |
| "unverified": { | |
| "type": "keyword" | |
| } | |
| } | |
| }, | |
| "contact": { | |
| "type": "object", | |
| "properties": { | |
| "original": { | |
| "type": "text" | |
| } | |
| } | |
| }, | |
| "middle": { | |
| "type": "object", | |
| "properties": { | |
| "probability": { | |
| "type": "half_float" | |
| } | |
| } | |
| }, | |
| "last_restarted": { | |
| "format": "strict_date_optional_time||epoch_millis||yyyy-MM-dd HH:mm:ss", | |
| "index": true, | |
| "ignore_malformed": false, | |
| "store": false, | |
| "type": "date", | |
| "doc_values": true | |
| }, | |
| "bandwidth": { | |
| "type": "object", | |
| "properties": { | |
| "rate": { | |
| "type": "long" | |
| }, | |
| "advertised": { | |
| "type": "long" | |
| }, | |
| "burst": { | |
| "type": "long" | |
| }, | |
| "observed": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "exit_policy": { | |
| "eager_global_ordinals": false, | |
| "index_phrases": false, | |
| "fielddata": false, | |
| "norms": true, | |
| "index": true, | |
| "store": false, | |
| "type": "text", | |
| "index_options": "positions" | |
| }, | |
| "version": { | |
| "type": "keyword" | |
| }, | |
| "overload": { | |
| "format": "strict_date_optional_time||epoch_millis||epoch_second", | |
| "index": true, | |
| "ignore_malformed": false, | |
| "store": false, | |
| "type": "date", | |
| "doc_values": true | |
| }, | |
| "consensus": { | |
| "type": "object", | |
| "properties": { | |
| "weight_fraction": { | |
| "type": "long" | |
| }, | |
| "weight": { | |
| "type": "long" | |
| } | |
| } | |
| }, | |
| "exit": { | |
| "type": "object", | |
| "properties": { | |
| "geo": { | |
| "type": "object", | |
| "properties": { | |
| "continent_name": { | |
| "type": "keyword" | |
| }, | |
| "region_iso_code": { | |
| "type": "keyword" | |
| }, | |
| "city_name": { | |
| "type": "keyword" | |
| }, | |
| "country_iso_code": { | |
| "type": "keyword" | |
| }, | |
| "country_name": { | |
| "type": "keyword" | |
| }, | |
| "location": { | |
| "type": "geo_point" | |
| }, | |
| "region_name": { | |
| "type": "keyword" | |
| } | |
| } | |
| }, | |
| "as": { | |
| "type": "object", | |
| "properties": { | |
| "number": { | |
| "type": "long" | |
| }, | |
| "organization": { | |
| "type": "object", | |
| "properties": { | |
| "name": { | |
| "type": "keyword" | |
| } | |
| } | |
| }, | |
| "network": { | |
| "type": "ip_range" | |
| } | |
| } | |
| }, | |
| "probability": { | |
| "type": "half_float" | |
| }, | |
| "ip": { | |
| "type": "ip" | |
| }, | |
| "policy": { | |
| "type": "object", | |
| "properties": { | |
| "ipv4": { | |
| "type": "object", | |
| "properties": { | |
| "reject": { | |
| "type": "text" | |
| }, | |
| "accept": { | |
| "type": "text" | |
| } | |
| } | |
| }, | |
| "ipv6": { | |
| "type": "object", | |
| "properties": { | |
| "reject": { | |
| "type": "text" | |
| }, | |
| "accept": { | |
| "type": "text" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "as": { | |
| "type": "object", | |
| "properties": { | |
| "number": { | |
| "type": "long" | |
| }, | |
| "organization": { | |
| "type": "object", | |
| "properties": { | |
| "name": { | |
| "type": "keyword" | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "version_status": { | |
| "type": "keyword" | |
| }, | |
| "name": { | |
| "type": "keyword" | |
| }, | |
| "detail": { | |
| "type": "object", | |
| "properties": { | |
| "BadExit": { | |
| "type": "text" | |
| }, | |
| "Unnamed": { | |
| "type": "text" | |
| }, | |
| "Authority": { | |
| "type": "text" | |
| }, | |
| "Valid": { | |
| "type": "text" | |
| }, | |
| "NoEdConsensus": { | |
| "type": "text" | |
| }, | |
| "Running": { | |
| "type": "text" | |
| }, | |
| "Stable": { | |
| "type": "text" | |
| }, | |
| "Guard": { | |
| "type": "text" | |
| }, | |
| "Fast": { | |
| "type": "text" | |
| }, | |
| "Exit": { | |
| "type": "text" | |
| }, | |
| "HSDir": { | |
| "type": "text" | |
| } | |
| } | |
| }, | |
| "recommended_version": { | |
| "type": "boolean" | |
| }, | |
| "family": { | |
| "type": "object", | |
| "properties": { | |
| "effective": { | |
| "eager_global_ordinals": false, | |
| "index_phrases": false, | |
| "fielddata": false, | |
| "norms": true, | |
| "index": true, | |
| "store": false, | |
| "type": "text", | |
| "index_options": "positions" | |
| }, | |
| "indirect": { | |
| "eager_global_ordinals": false, | |
| "index_phrases": false, | |
| "fielddata": false, | |
| "norms": true, | |
| "index": true, | |
| "store": false, | |
| "type": "text", | |
| "index_options": "positions" | |
| }, | |
| "alleged": { | |
| "eager_global_ordinals": false, | |
| "index_phrases": false, | |
| "fielddata": false, | |
| "norms": true, | |
| "index": true, | |
| "store": false, | |
| "type": "text", | |
| "index_options": "positions" | |
| } | |
| } | |
| }, | |
| "last_changed": { | |
| "format": "strict_date_optional_time||epoch_millis||yyyy-MM-dd HH:mm:ss", | |
| "index": true, | |
| "ignore_malformed": false, | |
| "store": false, | |
| "type": "date", | |
| "doc_values": true | |
| } | |
| } | |
| }, | |
| "threat": { | |
| "type": "object", | |
| "properties": { | |
| "indicator": { | |
| "type": "object", | |
| "properties": { | |
| "geo": { | |
| "type": "object", | |
| "properties": { | |
| "continent_name": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| }, | |
| "region_iso_code": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| }, | |
| "city_name": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| }, | |
| "country_iso_code": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| }, | |
| "country_name": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| }, | |
| "location": { | |
| "ignore_malformed": false, | |
| "type": "geo_point", | |
| "ignore_z_value": true | |
| }, | |
| "region_name": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "index_options": "docs", | |
| "split_queries_on_whitespace": false, | |
| "doc_values": true | |
| } | |
| } | |
| }, | |
| "as": { | |
| "type": "object", | |
| "properties": { | |
| "number": { | |
| "coerce": true, | |
| "index": true, | |
| "ignore_malformed": false, | |
| "store": false, | |
| "type": "long", | |
| "doc_values": true | |
| }, | |
| "organization": { | |
| "dynamic": true, | |
| "type": "object", | |
| "enabled": true, | |
| "properties": { | |
| "name": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| } | |
| }, | |
| "subobjects": true | |
| }, | |
| "network": { | |
| "coerce": true, | |
| "index": true, | |
| "store": false, | |
| "type": "ip_range" | |
| } | |
| } | |
| }, | |
| "first_seen": { | |
| "format": "strict_date_optional_time||epoch_millis||yyyy-MM-dd HH:mm:ss", | |
| "index": true, | |
| "ignore_malformed": false, | |
| "store": false, | |
| "type": "date", | |
| "doc_values": true | |
| }, | |
| "last_restarted": { | |
| "format": "strict_date_optional_time||epoch_millis||yyyy-MM-dd HH:mm:ss", | |
| "index": true, | |
| "ignore_malformed": false, | |
| "store": false, | |
| "type": "date", | |
| "doc_values": true | |
| }, | |
| "last_seen": { | |
| "format": "strict_date_optional_time||epoch_millis||yyyy-MM-dd HH:mm:ss", | |
| "index": true, | |
| "ignore_malformed": false, | |
| "store": false, | |
| "type": "date", | |
| "doc_values": true | |
| }, | |
| "ip": { | |
| "index": true, | |
| "store": false, | |
| "type": "ip", | |
| "doc_values": true | |
| }, | |
| "last_changed": { | |
| "format": "strict_date_optional_time||epoch_millis||yyyy-MM-dd HH:mm:ss", | |
| "index": true, | |
| "ignore_malformed": false, | |
| "store": false, | |
| "type": "date", | |
| "doc_values": true | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "event": { | |
| "type": "object", | |
| "properties": { | |
| "kind": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| }, | |
| "module": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| }, | |
| "category": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| }, | |
| "type": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| }, | |
| "dataset": { | |
| "eager_global_ordinals": false, | |
| "norms": false, | |
| "index": true, | |
| "store": false, | |
| "type": "keyword", | |
| "split_queries_on_whitespace": false, | |
| "index_options": "docs", | |
| "doc_values": true | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "lifecycle": { | |
| "enabled": true | |
| } | |
| }, | |
| "index_patterns": [ | |
| "logs-ti_tor.node_activity*" | |
| ], | |
| "data_stream": { | |
| "hidden": false, | |
| "allow_custom_routing": false | |
| }, | |
| "composed_of": [], | |
| "ignore_missing_component_templates": [], | |
| "allow_auto_create": true | |
| } | |
| # Navigate to Fleet, select the policy you wish to modify and add the Custom API Intergration | |
| # For Elastic-Agent Custom API integration fill in the following: | |
| Dataset name: | |
| ti_tor.node_activity | |
| Ingest Pipeline: | |
| logs-ti_tor.node_activity | |
| Request URL: | |
| https://onionoo.torproject.org/details?fields=exit_addresses,nickname,fingerprint,running,as_name,verified_host_names,unverified_host_names,or_addresses,last_seen,last_changed_address_or_port,first_seen,hibernating,last_restarted,bandwidth_rate,bandwidth_burst,observed_bandwidth,flags,version,version_status,advertised_bandwidth,platform,recommended_version,contact | |
| Request Interval: | |
| 60m | |
| Request HTTP Method: | |
| GET | |
| # SCROLL DOWN A BIT | |
| Response Split: | |
| target: body.relays | |
| # SCROLL DOWN SOME MORE AND EXPAND 'dvanced options' | |
| # CONTINUE SCROLLING | |
| Processors: | |
| - decode_json_fields: | |
| fields: ["message"] | |
| target: "" | |
| overwrite_keys: true | |
| - rename: | |
| fields: | |
| - from: "advertised_bandwidth" | |
| to: "tor.bandwidth.advertised" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "bandwidth_burst" | |
| to: "tor.bandwidth.burst" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "bandwidth_rate" | |
| to: "tor.bandwidth.rate" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "observed_bandwidth" | |
| to: "tor.bandwidth.observed" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "as" | |
| to: "tor.as.number" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "as_name" | |
| to: "tor.as.organization.name" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "consensus_weight" | |
| to: "tor.consensus.weight" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "consensus_weight_fraction" | |
| to: "tor.consensus.weight_fraction" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "contact" | |
| to: "tor.contact.original" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "alleged_family" | |
| to: "tor.family.alleged" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "effective_family" | |
| to: "tor.family.effective" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_addresses" | |
| to: "tor.exit.ip" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_policy" | |
| to: "tor.exit_policy" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_policy_summary.accept" | |
| to: "tor.exit.policy.ipv4.accept" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_policy_summary.reject" | |
| to: "tor.exit.policy.ipv4.reject" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_policy_v6_summary.accept" | |
| to: "tor.exit.policy.ipv6.accept" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_policy_v6_summary.reject" | |
| to: "tor.exit.policy.ipv6.reject" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_probability" | |
| to: "tor.exit.probability" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "fingerprint" | |
| to: "tor.fingerprint" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "first_seen" | |
| to: "tor.first_seen" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "flags" | |
| to: "tor.description" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "guard_probability" | |
| to: "tor.guard.probability" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "indirect_family" | |
| to: "tor.family.indirect" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "last_changed_address_or_port" | |
| to: "tor.last_changed" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "last_restarted" | |
| to: "tor.last_restarted" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "last_seen" | |
| to: "tor.last_seen" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "measured" | |
| to: "tor.measured" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "middle_probability" | |
| to: "tor.middle.probability" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "nickname" | |
| to: "tor.name" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "or_addresses" | |
| to: "tor.routing.addresses" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "overload_general_timestamp" | |
| to: "tor.overload" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "platform" | |
| to: "tor.platform" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "recommended_version" | |
| to: "tor.recommended_version" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "running" | |
| to: "tor.running" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "unverified_host_names" | |
| to: "tor.hostname.unverified" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "verified_host_names" | |
| to: "tor.hostname.verified" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "version" | |
| to: "tor.version" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "version_status" | |
| to: "tor.version_status" | |
| ignore_missing: true | |
| - add_fields: | |
| target: event | |
| fields: | |
| module: "ti_tor" | |
| category: "threat" | |
| type: "indicator" | |
| kind: "enrichment" | |
| dataset: "node_activity" | |
| - copy_fields: | |
| fields: | |
| - from: "tor.exit.ip" | |
| to: "threat.indicator.ip" | |
| fail_on_error: false | |
| ignore_missing: true | |
| # IF RUNNING FILEBEAT: | |
| Use this config instead and restart filebeat: | |
| filebeat.inputs: | |
| # TOR NODES ACTIVITY | |
| - type: httpjson | |
| interval: 50m | |
| index: logs-ti_tor.node_activity | |
| pipeline: "logs-ti_tor.node_activity" | |
| request.url: https://onionoo.torproject.org/details?fields=exit_addresses,nickname,fingerprint,running,as_name,verified_host_names,unverified_host_names,or_addresses,last_seen,last_changed_address_or_port,first_seen,hibernating,last_restarted,bandwidth_rate,bandwidth_burst,observed_bandwidth,flags,version,version_status,advertised_bandwidth,platform,recommended_version,contact | |
| response.split: | |
| target: body.relays | |
| # PROCESSORS | |
| processors: | |
| - decode_json_fields: | |
| fields: ["message"] | |
| target: "" | |
| overwrite_keys: true | |
| - rename: | |
| fields: | |
| - from: "advertised_bandwidth" | |
| to: "tor.bandwidth.advertised" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "bandwidth_burst" | |
| to: "tor.bandwidth.burst" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "bandwidth_rate" | |
| to: "tor.bandwidth.rate" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "observed_bandwidth" | |
| to: "tor.bandwidth.observed" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "as" | |
| to: "tor.as.number" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "as_name" | |
| to: "tor.as.organization.name" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "consensus_weight" | |
| to: "tor.consensus.weight" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "consensus_weight_fraction" | |
| to: "tor.consensus.weight_fraction" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "contact" | |
| to: "tor.contact.original" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "alleged_family" | |
| to: "tor.family.alleged" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "effective_family" | |
| to: "tor.family.effective" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_addresses" | |
| to: "tor.exit.ip" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_policy" | |
| to: "tor.exit_policy" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_policy_summary.accept" | |
| to: "tor.exit.policy.ipv4.accept" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_policy_summary.reject" | |
| to: "tor.exit.policy.ipv4.reject" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_policy_v6_summary.accept" | |
| to: "tor.exit.policy.ipv6.accept" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_policy_v6_summary.reject" | |
| to: "tor.exit.policy.ipv6.reject" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "exit_probability" | |
| to: "tor.exit.probability" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "fingerprint" | |
| to: "tor.fingerprint" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "first_seen" | |
| to: "tor.first_seen" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "flags" | |
| to: "tor.description" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "guard_probability" | |
| to: "tor.guard.probability" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "indirect_family" | |
| to: "tor.family.indirect" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "last_changed_address_or_port" | |
| to: "tor.last_changed" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "last_restarted" | |
| to: "tor.last_restarted" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "last_seen" | |
| to: "tor.last_seen" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "measured" | |
| to: "tor.measured" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "middle_probability" | |
| to: "tor.middle.probability" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "nickname" | |
| to: "tor.name" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "or_addresses" | |
| to: "tor.routing.addresses" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "overload_general_timestamp" | |
| to: "tor.overload" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "platform" | |
| to: "tor.platform" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "recommended_version" | |
| to: "tor.recommended_version" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "running" | |
| to: "tor.running" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "unverified_host_names" | |
| to: "tor.hostname.unverified" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "verified_host_names" | |
| to: "tor.hostname.verified" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "version" | |
| to: "tor.version" | |
| ignore_missing: true | |
| - rename: | |
| fields: | |
| - from: "version_status" | |
| to: "tor.version_status" | |
| ignore_missing: true | |
| - add_fields: | |
| target: event | |
| fields: | |
| module: "ti_tor" | |
| category: "threat" | |
| type: "indicator" | |
| kind: "enrichment" | |
| dataset: "node_activity" | |
| - copy_fields: | |
| fields: | |
| - from: tor.exit.ip | |
| to: threat.indicator.ip | |
| fail_on_error: false | |
| ignore_missing: true |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment