gistfile1.txt

---
replicas: 2

# Allows you to add any config files in /usr/share/logstash/config/
# such as logstash.yml and log4j2.properties
#
# Note that when overriding logstash.yml, `http.host: 0.0.0.0` should always be included
# to make default probes work.
logstashConfig:
  logstash.yml: |
    http.host: "0.0.0.0"
    xpack.monitoring.enabled: true
    xpack.monitoring.elasticsearch.hosts: [ "https://logsys-ingest.elasticsearch:9200" ]
    xpack.monitoring.elasticsearch.username: "beats_system"
    xpack.monitoring.elasticsearch.password: "mypassword"
    xpack.monitoring.elasticsearch.ssl.certificate_authority: "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
    queue.type: persisted
    queue.max_bytes: 200gb 

# Allows you to add any pipeline files in /usr/share/logstash/pipeline/
### ***warn*** there is a hardcoded logstash.conf in the image, override it first
logstashPipeline: 
  logstash.conf: |
    input {
    	syslog {
    		timezone => "America/Sao_Paulo"
    		port => "5514"
    		type => "syslog"
    		tags => [ "PAN-OS_SysLog" ]
    	}
    }
    filter {
    	# Log types are "TRAFFIC", "THREAT", "CONFIG" and "SYSTEM". URL & Wildfire logs are inside Threat logs
    	# Log fields: https://www.paloaltonetworks.com/documentation/80/pan-os/pan-os/monitoring/syslog-field-descriptions
    	if ([message] =~ /TRAFFIC/) {
    		csv {
    			source => "message"
    			columns => [
    				"FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Threat_ContentType", "FUTURE_USE",
    				"GeneratedTime", "SourceIP", "DestinationIP", "NATSourceIP", "NATDestinationIP", "RuleName",
    				"SourceUser", "DestinationUser", "Application", "VirtualSystem", "SourceZone", "DestinationZone",
    				"InboundInterface", "OutboundInterface", "LogForwardingProfile", "TimeLogged", "SessionID",
    				"RepeatCount", "SourcePort", "DestinationPort", "NATSourcePort", "NATDestinationPort", "Flags",
    				"Protocol", "Action", "Bytes", "BytesSent", "BytesReceived", "Packets", "StartTime", "ElapsedTime",
    				"URLCategory", "FUTURE_USE", "SequenceNumber", "ActionFlags", "SourceLocation",
    				"DestinationLocation", "FUTURE_USE", "PacketsSent", "PacketsReceived", "SessionEndReason",
    				"DeviceGroupHierarchyLevel1", "DeviceGroupHierarchyLevel2", "DeviceGroupHierarchyLevel3",
    				"DeviceGroupHierarchyLevel4", "VirtualSystemName", "DeviceName", "ActionSource", "SourceVMUUID",
    				"DestinationVMUUID", "TunnelID_IMSI", "MonitorTag_IMEI", "ParentSessionID", "ParentStartTime",
    				"TunnelType"
    			]
    		}
    		mutate { rename => [ "[host]", "[host][name]"] }
    
    		mutate {
    			convert => [ "Bytes", "integer" ]
    			convert => [ "BytesReceived", "integer" ]
    			convert => [ "BytesSent", "integer" ]
    			convert => [ "ElapsedTime", "integer" ]
    			convert => [ "GeoIP.dma_code", "integer" ]
    			convert => [ "GeoIP.latitude", "float" ]
    			convert => [ "GeoIP.longitude", "float" ]
    			convert => [ "NATDestinationPort", "integer" ]
    			convert => [ "NATSourcePort", "integer" ]
    			convert => [ "Packets", "integer" ]
    			convert => [ "PacketsReceived", "integer" ]
    			convert => [ "PacketsSent", "integer" ]
    			convert => [ "SequenceNumber", "integer" ]
    
    			add_tag => [ "PAN-OS_Traffic"]
    		}
    	}
    	else if ([message] =~ /HIPMATCH/) {
    		csv {
    			source => "message"
    			columns => [
    				"FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Threat_ContentType", "FUTURE_USE",
    				"GeneratedTime", "SourceUser", "VirtualSystem", "MachineName", "OperatingSystem",
    				"SourceAddress", "HIP", "RepeatCount", "HIPType", "FUTURE_USE", "FUTURE_USE", "SequenceNumber",
    				"ActionFlags", "DeviceGroupHierarchyLevel1", "DeviceGroupHierarchyLevel2",
    				"DeviceGroupHierarchyLevel3", "DeviceGroupHierarchyLevel4", "VirtualSystemName",
    				"DeviceName", "VirtualSystemID", "IPv6SourceAddress", "HostID"
    			]
    		}
    			mutate { rename => [ "[host]", "[host][name]"] }
    
    			mutate {
    				convert => [ "GeoIP.dma_code", "integer" ]
    				convert => [ "GeoIP.latitude", "float" ]
    				convert => [ "GeoIP.longitude", "float" ]
    				add_tag => [ "PAN-OS_HIPMATCH"]
    		}
    	}
    	else if ([message] =~ /THREAT/) {
    		csv {
    			source => "message"
    			columns => [
    				"FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Threat_ContentType", "FUTURE_USE",
    				"GeneratedTime", "SourceIP", "DestinationIP", "NATSourceIP", "NATDestinationIP", "RuleName",
    				"SourceUser", "DestinationUser", "Application", "VirtualSystem", "SourceZone", "DestinationZone",
    				"InboundInterface", "OutboundInterface", "LogForwardingProfile", "FUTURE_USE", "SessionID",
    				"RepeatCount", "SourcePort", "DestinationPort", "NATSourcePort", "NATDestinationPort", "Flags",
    				"Protocol", "Action", "Miscellaneous", "ThreatID", "URLCategory", "Severity", "Direction",
    				"SequenceNumber", "ActionFlags", "SourceLocation", "DestinationLocation", "FUTURE_USE",
    				"ContentType", "PCAP_ID", "FileDigest", "Cloud", "URLIndex", "UserAgent", "FileType",
    				"X-Forwarded-For", "Referer", "Sender", "Subject", "Recipient", "ReportID",
    				"DeviceGroupHierarchyLevel1", "DeviceGroupHierarchyLevel2", "DeviceGroupHierarchyLevel3",
    				"DeviceGroupHierarchyLevel4", "VirtualSystemName", "DeviceName", "FUTURE_USE", "SourceVMUUID",
    				"DestinationVMUUID", "HTTPMethod", "TunnelID_IMSI", "MonitorTag_IMEI", "ParentSessionID",
    				"ParentStartTime", "TunnelType", "ThreatCategory", "ContentVersion", "FUTURE_USE"
    			]
    		}
    		mutate { rename => [ "[host]", "[host][name]"] }
    		mutate {
    			convert => [ "GeoIP.dma_code", "integer" ]
    			convert => [ "GeoIP.latitude", "float" ]
    			convert => [ "GeoIP.longitude", "float" ]
    			convert => [ "NATDestinationPort", "integer" ]
    			convert => [ "NATSourcePort", "integer" ]
    			convert => [ "SequenceNumber", "integer" ]
    			add_tag => ["PAN-OS_Threat"]
    		}
    	}
    
    	else if ([message] =~ /CONFIG/) {
    		csv {
    			source => "message"
    			columns => [
    				"FUTURE_USE", "Receive Time", "Serial Number", "Type", "Subtype", "FUTURE_USE", "Generated Time", "Host",
    				"Virtual System", "Command", "Admin", "Client", "Result", "Configuration Path", "Before Change Detail",
    				"After Change Detail", "Sequence Number", "Action Flags", "Device Group Hierarchy Level 1",
    				"Device Group Hierarchy Level 2", "Device Group Hierarchy Level 3", "Device Group Hierarchy Level 4",
    				"Virtual System Name", "Device Name"
    			]
    		}
    		mutate { add_tag => [ "PAN-OS_Config"] }
    		mutate { rename => [ "[host]", "[host][name]"] }
    	}
    
    	else if ([message] =~ /SYSTEM/) {
    		csv {
    			source => "message"
    			columns => [
    				"FUTURE_USE", "Receive Time", "Serial Number", "Type", "Content/Threat Type", "FUTURE_USE", "Generated Time",
    				"Virtual System", "Event ID", "Object", "FUTURE_USE", "FUTURE_USE", "Module", "Severity", "Description",
    				"Sequence Number", "Action Flags", "Device Group Hierarchy Level 1", "Device Group Hierarchy Level 2",
    				"Device Group Hierarchy Level 3", "Device Group Hierarchy Level 4", "Virtual System Name", "Device Name"
    			]
    		}
    		mutate { add_tag => [ "PAN-OS_System"] }
    		mutate { rename => [ "[host]", "[host][name]"] }
    		if "globalprotect" in [Content/Threat Type]{
    			mutate {
    				copy => { "Description" => "tmp_field1" }
    			}
    			grok {
    				match => { "tmp_field1" => "GlobalProtect (?<GlobalProtect.Action>[^\.]*)\. %{GREEDYDATA:tmp_field2}" }
    			}
    			kv {
    				source => "tmp_field2"
    				target => "GlobalProtect"
    				trim_key => " "
    				trim_value => " ."
    				field_split => ","
    				value_split => ":"
    			}
    			mutate {
    				remove_field => ["tmp_field1"]
    				remove_field => ["tmp_field2"]
    			}
    			geoip {
    				source => "[GlobalProtect][Login from]"
    				target => "[GlobalProtect][Login From][GeoIP]"
    			}
    		}
    	}
    
    	mutate {
    		# Original message has been fully parsed, so remove it.
    		remove_field => [ "message" ]
    	}
    
    	# Geolocate logs that have SourceIP if that SourceIP is a non-RFC1918 address
    	if [SourceIP] and [SourceIP] !~ "(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)|(^169\.254\.)" {
    		geoip {
    			source => "SourceIP"
    			target => "SourceIPGeo"
    		}
    
    		# Delete 0,0 in SourceIPGeo.location if equal to 0,0
    		if ([SourceIPGeo.location] and [SourceIPGeo.location] =~ "0,0") {
    			mutate {
    				replace => [ "SourceIPGeo.location", "" ]
    			}
    		}
    	}
    
    	# Geolocate logs that have DestinationIP and if that DestinationIP is a non-RFC1918 address
    	if [DestinationIP] and [DestinationIP] !~ "(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)|(^169\.254\.)" {
    		geoip {
    			source => "DestinationIP"
    			target => "DestinationIPGeo"
    		}
    
    		# Delete 0,0 in DestinationIPGeo.location if equal to 0,0
    		if ([DestinationIPGeo.location] and [DestinationIPGeo.location] =~ "0,0") {
    			mutate {
    				replace => [ "DestinationIPGeo.location", "" ]
    			}
    		}
    	}
    
    	# Takes the 5-tuple of source address, source port, destination address, destination port, and protocol and does a SHA1 hash to fingerprint the flow.This is a useful
    	# way to be able to do top N terms queries on flows, not just on one field.
    	if [SourceIP] and [DestinationIP] {
    		fingerprint {
    			concatenate_sources => true
    			method => "SHA1"
    			key => "logstash"
    			source => [ "SourceIP", "SourcePort", "DestinationIP", "DestinationPort", "Protocol" ]
    		}
    	}
    }
    
    output {
    	if "PAN-OS_Traffic" in [tags] {
    		elasticsearch {
    			hosts => "logsys-ingest.elasticsearch:9200"
    			ssl => true
    			cacert => "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
    			user => "${ELASTICSEARCH_USERNAME}"
    			password => "${ELASTICSEARCH_PASSWORD}"
    			ssl_certificate_verification => true
    			#index => "panos-traffic-%{+YYYY.MM.dd}"
    			#ecs_compatibility => v1
    			data_stream => "true"
    			data_stream_sync_fields => true
    			data_stream_dataset => "panos"
    			data_stream_namespace => "traffic"
    		}
    	}
    	else if "PAN-OS_Threat" in [tags] {
    		elasticsearch {
    			hosts => "logsys-ingest.elasticsearch:9200"
    			ssl => true
    			cacert => "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
    			user => "${ELASTICSEARCH_USERNAME}"
    			password => "${ELASTICSEARCH_PASSWORD}"
    			ssl_certificate_verification => true
    			#index => "panos-threat-%{+YYYY.MM.dd}"
    			#ecs_compatibility => v1
    			data_stream => "true"
    			data_stream_sync_fields => true
    			data_stream_dataset => "panos"
    			data_stream_namespace => "threat"
     		}
    	}
    	else if "PAN-OS_Config" in [tags] {
    		elasticsearch {
    			hosts => "logsys-ingest.elasticsearch:9200"
    			ssl => true
    			cacert => "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
    			user => "${ELASTICSEARCH_USERNAME}"
    			password => "${ELASTICSEARCH_PASSWORD}"
    			ssl_certificate_verification => true
    			#index => "panos-config-%{+YYYY.MM.dd}"
    			#ecs_compatibility => v1
    			data_stream => "true"
    			data_stream_sync_fields => true
    			data_stream_dataset => "panos"
    			data_stream_namespace => "config"
     		}
    	}
    	else if "PAN-OS_System" in [tags] {
    		elasticsearch {
    			hosts => "logsys-ingest.elasticsearch:9200"
    			ssl => true
    			cacert => "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
    			user => "${ELASTICSEARCH_USERNAME}"
    			password => "${ELASTICSEARCH_PASSWORD}"
    			ssl_certificate_verification => true
    			#index => "panos-system-%{+YYYY.MM.dd}"
    			#ecs_compatibility => v1
    			data_stream => "true"
    			data_stream_sync_fields => true
    			data_stream_dataset => "panos"
    			data_stream_namespace => "system"
     		}
    	}
    	else if "PAN-OS_HIPMATCH" in [tags] {
    		elasticsearch {
    			hosts => "logsys-ingest.elasticsearch:9200"
    			ssl => true
    			cacert => "/usr/share/elasticsearch/config/certs/elastic-certificate.crt"
    			user => "${ELASTICSEARCH_USERNAME}"
    			password => "${ELASTICSEARCH_PASSWORD}"
    			ssl_certificate_verification => true
    			#index => "panos-hip-%{+YYYY.MM.dd}"
    			#ecs_compatibility => v1
    			data_stream => "true"
    			data_stream_sync_fields => true
    			data_stream_dataset => "panos"
    			data_stream_namespace => "hip"
     		}
    	}
    	#stdout {
    	#codec => rubydebug
    	#}
    }

# Extra environment variables to append to this nodeGroup
# This will be appended to the current 'env:' key. You can use any of the kubernetes env
# syntax here
extraEnvs: 
  - name: 'ELASTICSEARCH_USERNAME'
    valueFrom:
      secretKeyRef:
        name: elastic-credentials
        key: username
  - name: 'ELASTICSEARCH_PASSWORD'
    valueFrom:
      secretKeyRef:
        name: elastic-credentials
        key: password

# A list of secrets and their paths to mount inside the pod
secretMounts: 
  - name: elastic-certificates
    secretName: elastic-certificate-crt
    path: /usr/share/elasticsearch/config/certs

logstashJavaOpts: "-Xmx2g -Xms2g"

resources:
  requests:
    cpu: "1000m"
    memory: "2Gi"
  limits:
    cpu: "3"
    memory: "4Gi"

volumeClaimTemplate:
  accessModes: [ "ReadWriteOnce" ]
  resources:
    requests:
      storage: 210Gi

persistence:
  enabled: true
  annotations: {}

service: 
  type: NodePort
  ports:
    - name: paloalto-input
      port: 5514
      protocol: TCP
      nodePort: 30001
    - name: paloalto-udp
      port: 5514
      protocol: UDP
      nodePort: 30001
ingress:
  enabled: false