Elastiflow

docker-compose.yml

version: '3'
services:
  kibana:
    image: docker.elastic.co/kibana/kibana:7.13.1
    restart: unless-stopped
    hostname: NODE_NAME
    network_mode: bridge
    ports:
      # HTTP/REST
      - 5601:5601/tcp
    environment:
      TELEMETRY_OPTIN: 'false'
      TELEMETRY_ENABLED: 'false'
      NEWSFEED_ENABLED: 'false'

      SERVER_NAME: 'NODE_NAME'
      SERVER_HOST: '0.0.0.0'
      SERVER_PORT: 5601
      SERVER_MAXPAYLOADBYTES: 8388608

      ELASTICSEARCH_HOSTS: 'http://192.0.2.11:9200'
      ELASTICSEARCH_USERNAME: 'kibana_system'
      ELASTICSEARCH_PASSWORD: 'CHANGEME'
      ELASTICSEARCH_REQUESTTIMEOUT: 132000
      ELASTICSEARCH_SHARDTIMEOUT: 120000

      #ELASTICSEARCH_SSL_CERTIFICATE: /etc/kibana/certs/node/node.crt
      #ELASTICSEARCH_SSL_KEY: /etc/kibana/certs/node/node.key
      #ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: /etc/kibana/certs/ca/ca.crt
      ELASTICSEARCH_SSL_VERIFICATIONMODE: 'none' 

      KIBANA_AUTOCOMPLETETIMEOUT: 3000
      KIBANA_AUTOCOMPLETETERMINATEAFTER: 2500000

      VIS_TYPE_VEGA_ENABLEEXTERNALURLS: 'true'

      XPACK_MAPS_SHOWMAPVISUALIZATIONTYPES: 'true'
      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'ElastiFlow_0123456789_0123456789_0123456789'
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: NODE_NAME
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - ./var/lib/elasticsearch:/usr/share/elasticsearch/data
      - ./etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms12g -Xmx12g'

      cluster.name: elastiflow
      node.name: NODE_NAME

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.11
      http.port: 9200
      http.publish_port: 9200

      discovery.type: 'single-node'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

#      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
#      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
#      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
#      xpack.security.http.ssl.verification_mode: 'none'
#      xpack.security.http.ssl.enabled: 'true'

#      xpack.monitoring.collection.enabled: 'true'
#      xpack.monitoring.collection.interval: 30s

#      xpack.security.enabled: 'true'
#      xpack.security.audit.enabled: 'false'
  # ElastiFlow Unified Flow Collector
  flow-collector:
    image: elastiflow/flow-collector:5.1.6
    container_name: flow-collector
    restart: 'unless-stopped'
    network_mode: 'host'
    volumes:
      - ./etc/elastiflow:/etc/elastiflow
    environment:
      #EF_FLOW_ACCOUNT_ID: ''
      #EF_FLOW_LICENSE_KEY: ''
      #EF_FLOW_LICENSED_CORES: 

      #EF_FLOW_LOGGER_LEVEL: 'info'
      #EF_FLOW_LOGGER_ENCODING: 'json'
      #EF_FLOW_LOGGER_FILE_LOG_ENABLE: 'false'
      #EF_FLOW_LOGGER_FILE_LOG_DIR: '/var/log/elastiflow/flowcoll'
      #EF_FLOW_LOGGER_FILE_LOG_COUNT: 4
      #EF_FLOW_LOGGER_FILE_LOG_INTERVAL: 'daily'
      #EF_FLOW_LOGGER_FILE_LOG_SIZE: '100MB'

      EF_FLOW_SERVER_UDP_IP: '0.0.0.0'
      EF_FLOW_SERVER_UDP_PORT: 2055
      #EF_FLOW_SERVER_UDP_PACKET_STREAM_MAX_SIZE: 
      #EF_FLOW_SERVER_UDP_READ_BUFFER_MAX_SIZE: 33554432

      #EF_FLOW_DECODER_SETTINGS_PATH: '/etc/elastiflow'

      #EF_FLOW_DECODER_IPFIX_ENABLE: 'true'
      #EF_FLOW_DECODER_NETFLOW1_ENABLE: 'true'
      #EF_FLOW_DECODER_NETFLOW5_ENABLE: 'true'
      #EF_FLOW_DECODER_NETFLOW6_ENABLE: 'true'
      #EF_FLOW_DECODER_NETFLOW7_ENABLE: 'true'
      #EF_FLOW_DECODER_NETFLOW9_ENABLE: 'true'
      #EF_FLOW_DECODER_SFLOW5_ENABLE: 'true'
      #EF_FLOW_DECODER_SFLOW_FLOWS_ENABLE: 'true'
      #EF_FLOW_DECODER_SFLOW_FLOWS_KEEP_SAMPLES: 'false'
      #EF_FLOW_DECODER_SFLOW_COUNTERS_ENABLE: 'true'

      #EF_FLOW_DECODER_TRANSLATE_KEEP_IDS: 'default'

      EF_FLOW_DECODER_ENRICH_DNS_ENABLE: 'true'
      EF_FLOW_DECODER_ENRICH_DNS_NAMESERVER_IP: '158.51.134.53'
      #EF_FLOW_DECODER_ENRICH_DNS_NAMESERVER_TIMEOUT: 
      #EF_FLOW_DECODER_ENRICH_DNS_CACHE_SIZE: 524288
      EF_FLOW_DECODER_ENRICH_DNS_RESOLVE_EXPORTER: 'true'
      EF_FLOW_DECODER_ENRICH_DNS_RESOLVE_PRIVATE: 'true'
      EF_FLOW_DECODER_ENRICH_DNS_RESOLVE_PUBLIC: 'true'
      #EF_FLOW_DECODER_ENRICH_DNS_USERDEF_ENABLE: 'false'
      #EF_FLOW_DECODER_ENRICH_DNS_USERDEF_PATH: 'settings/hostnames_user_defined.yml'

      #EF_FLOW_DECODER_ENRICH_NETIF_GET_ATTRS: 'true'
      #EF_FLOW_DECODER_ENRICH_NETIF_CACHE_SIZE: 262144

      #EF_FLOW_DECODER_ENRICH_SNMP_ENABLE: 'false'
      #EF_FLOW_DECODER_ENRICH_SNMP_PORT: 161
      #EF_FLOW_DECODER_ENRICH_SNMP_VERSION: 2
      #EF_FLOW_DECODER_ENRICH_SNMP_COMMUNITY: 'public'
      #EF_FLOW_DECODER_ENRICH_SNMP_TIMEOUT: 2
      #EF_FLOW_DECODER_ENRICH_SNMP_RETRIES: 1

      #EF_FLOW_DECODER_ENRICH_APP_CACHE_SIZE: 262144
      #EF_FLOW_DECODER_ENRICH_APP_USERDEF_ENABLE: 'false'
      #EF_FLOW_DECODER_ENRICH_APP_USERDEF_PRIVATE: 'true'
      #EF_FLOW_DECODER_ENRICH_APP_USERDEF_PUBLIC: 'true'
      #EF_FLOW_DECODER_ENRICH_APP_USERDEF_PATH: 'settings/apps_user_defined.yml'

      #EF_FLOW_DECODER_ENRICH_ASN_PREF: 'lookup'

      #EF_FLOW_DECODER_ENRICH_RISKIQ_ASN_ENABLE: 'false'
      #EF_FLOW_DECODER_ENRICH_RISKIQ_ASN_ENDPOINT: 'https://api.passivetotal.org/v2/netflow/as/download'
      #EF_FLOW_DECODER_ENRICH_RISKIQ_ASN_REFRESH_INTERVAL: 1440
      #EF_FLOW_DECODER_ENRICH_RISKIQ_THREAT_ENABLE: 'false'
      #EF_FLOW_DECODER_ENRICH_RISKIQ_THREAT_ENDPOINT: 'https://api.passivetotal.org/v2/netflow/blocklist/download'
      #EF_FLOW_DECODER_ENRICH_RISKIQ_THREAT_REFRESH_INTERVAL: 1440
      #EF_FLOW_DECODER_ENRICH_RISKIQ_API_USER: ''
      #EF_FLOW_DECODER_ENRICH_RISKIQ_API_KEY: ''
      #EF_FLOW_DECODER_ENRICH_RISKIQ_API_TIMEOUT: 30

      EF_FLOW_DECODER_ENRICH_MAXMIND_ASN_ENABLE: 'true'
      EF_FLOW_DECODER_ENRICH_MAXMIND_ASN_CACHE_SIZE: 262144
      EF_FLOW_DECODER_ENRICH_MAXMIND_ASN_PATH: 'maxmind/GeoLite2-ASN.mmdb'

      EF_FLOW_DECODER_ENRICH_MAXMIND_GEOIP_ENABLE: 'true'
      EF_FLOW_DECODER_ENRICH_MAXMIND_GEOIP_CACHE_SIZE: 262144
      EF_FLOW_DECODER_ENRICH_MAXMIND_GEOIP_PATH: 'maxmind/GeoLite2-City.mmdb'
      EF_FLOW_DECODER_ENRICH_MAXMIND_GEOIP_VALUES: 'city,country,country_code,location,timezone'
      EF_FLOW_DECODER_ENRICH_MAXMIND_GEOIP_LANG: 'en'

      #EF_FLOW_DECODER_ENRICH_SAMPLERATE_CACHE_SIZE: 32768
      #EF_FLOW_DECODER_ENRICH_SAMPLERATE_USERDEF_ENABLE: 'false'
      #EF_FLOW_DECODER_ENRICH_SAMPLERATE_USERDEF_PATH: 'settings/sample_rate.yml'

      #EF_FLOW_DECODER_ENRICH_COMMUNITYID_ENABLE: 'true'
      #EF_FLOW_DECODER_ENRICH_COMMUNITYID_SEED: 0
      #EF_FLOW_DECODER_ENRICH_CONVERSATIONID_ENABLE: 'true'
      #EF_FLOW_DECODER_ENRICH_CONVERSATIONID_SEED: 0

      #EF_FLOW_DECODER_ENRICH_JOIN_ASN: 'true'
      #EF_FLOW_DECODER_ENRICH_JOIN_GEOIP: 'true'
      #EF_FLOW_DECODER_ENRICH_JOIN_SEC: 'true'
      #EF_FLOW_DECODER_ENRICH_JOIN_NETATTR: 'true'

      #EF_FLOW_DECODER_DURATION_PRECISION: 'ms'
      #EF_FLOW_DECODER_TIMESTAMP_PRECISION: 'ms'
      #EF_FLOW_DECODER_PERCENT_NORM: 100
      #EF_FLOW_DECODER_ENRICH_EXPAND_CLISRV: 'true'
      #EF_FLOW_DECODER_ENRICH_KEEP_CPU_TICKS: 'false'

      #EF_FLOW_RECORD_STREAM_MAX_SIZE: 

      # stdout
      #EF_FLOW_OUTPUT_STDOUT_ENABLE: 'false'
      #EF_FLOW_OUTPUT_STDOUT_FORMAT: 'json_pretty'

      # monitor
      #EF_FLOW_OUTPUT_MONITOR_ENABLE: 'false'
      #EF_FLOW_OUTPUT_MONITOR_INTERVAL: 300

      # Elasticsearch
      EF_FLOW_OUTPUT_ELASTICSEARCH_ENABLE: 'true'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_ECS_ENABLE: 'false'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_BATCH_DEADLINE: 2000
      #EF_FLOW_OUTPUT_ELASTICSEARCH_BATCH_MAX_BYTES: 8388608
      #EF_FLOW_OUTPUT_ELASTICSEARCH_TIMESTAMP_SOURCE: 'end'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_PERIOD: 'daily'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_SUFFIX: ''

      #EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_ENABLE: 'true'
      EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_OVERWRITE: 'true'
      EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_SHARDS: 1
      EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_REPLICAS: 0
      #EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_REFRESH_INTERVAL: '10s'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_CODEC: 'best_compression'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_ILM_LIFECYCLE: ''
      #EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_ILM_ROLLOVER_ALIAS: ''
      #EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_ISM_POLICY: ''
      #EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_PIPELINE_DEFAULT: '_none'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_PIPELINE_FINAL: '_none'

      # A comma separated list of Elasticsearch nodes to use. DO NOT include "http://" or "https://"
      EF_FLOW_OUTPUT_ELASTICSEARCH_ADDRESSES: '192.0.2.11:9200'
      EF_FLOW_OUTPUT_ELASTICSEARCH_USERNAME: 'elastic'
      EF_FLOW_OUTPUT_ELASTICSEARCH_PASSWORD: 'changeme'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_CLOUD_ID: ''
      #EF_FLOW_OUTPUT_ELASTICSEARCH_API_KEY: ''

      #EF_FLOW_OUTPUT_ELASTICSEARCH_TLS_ENABLE: 'false'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_TLS_SKIP_VERIFICATION: 'false'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_TLS_CA_CERT_FILEPATH: '/etc/elastiflow/certs/ca/ca.crt'

      #EF_FLOW_OUTPUT_ELASTICSEARCH_RETRY_ENABLE: 'true'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_RETRY_ON_TIMEOUT_ENABLE: 'true'
      #EF_FLOW_OUTPUT_ELASTICSEARCH_MAX_RETRIES: 3
      #EF_FLOW_OUTPUT_ELASTICSEARCH_RETRY_BACKOFF: 1000
      
      # RiskIQ
      #EF_FLOW_OUTPUT_RISKIQ_ENABLE: 'false'
      #EF_FLOW_OUTPUT_RISKIQ_HOST: ''
      #EF_FLOW_OUTPUT_RISKIQ_PORT: 
      #EF_FLOW_OUTPUT_RISKIQ_CUSTOMER_UUID: ''
      #EF_FLOW_OUTPUT_RISKIQ_CUSTOMER_ENCRYPTION_KEY: ''