Minimal Elastic monitoring stack with enabled xpack-security

README.md

Minimal Elastic monitoring stack

You may use t3.medium AWS EC2 instance to run this stack.

Install docker runtime

$ sudo curl -fsSL https://get.docker.com -o get-docker.sh
$ sudo sh get-docker.sh
$ sudo usermod -a -G docker $USER

Install docker-compose binary

$ sudo curl -fsSL "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
$ sudo chmod +x /usr/local/bin/docker-compose

Increase allowed virtual memory

$ sudo sysctl -w vm.max_map_count=262144

Prepare directory

$ mkdir monitoring
$ cd monitoring

Prepare environment configuration for docker-compose

$ cat <<'EOF' > .env
COMPOSE_PROJECT_NAME=es 
CERTS_DIR=/usr/share/elasticsearch/config/certificates 
ELASTIC_PASSWORD=PleaseChangeMe
EOF

Replace PleaseChangeMe with something else

Prepare list of instances

$ cat <<'EOF' > instances.yml
instances:
- name: es01
  dns:
  - es01 
  - localhost
  ip:
  - 127.0.0.1
- name: es02
  dns:
  - es02
  - localhost
  ip:
  - 127.0.0.1
- name: kib01
  dns:
  - kib01
  - localhost
  ip:
  - 127.0.0.1
EOF

We will use this file to generate x509 certificates

Generate certificates

Prepare docker-compose file

$ cat <<'EOF' > create-certs.yml
version: '2.2'

services:
  create_certs:
    container_name: create_certs
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    command: >
      bash -c '
        if [[ ! -f /certs/bundle.zip ]]; then
          bin/elasticsearch-certutil cert --silent --pem --in config/certificates/instances.yml -out /certs/bundle.zip;
          unzip /certs/bundle.zip -d /certs; 
        fi;
        chown -R 1000:0 /certs
      '
    user: "0"
    working_dir: /usr/share/elasticsearch
    volumes: ['certs:/certs', '.:$CERTS_DIR']

volumes: {"certs"}
EOF

Generate certificates

$ docker-compose -f create-certs.yml run --rm create_certs

Run Elastic and Kibana

Prepare docker-compose file

$ cat <<'EOF' > docker-compose.yml
version: '2.2'

services:
  es01:
    container_name: es01
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    environment:
      - node.name=es01
      - http.port=9243
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - xpack.license.self_generated.type=trial 
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es01/es01.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate 
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key
      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data01:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    ports:
      - 9243:9243
    healthcheck:
      test: curl --silent --cacert $CERTS_DIR/ca/ca.crt https://localhost:9243 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - elastic

  es02:
    container_name: es02
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    environment:
      - node.name=es02
      - http.port=9243
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - xpack.license.self_generated.type=trial
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es02/es02.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate 
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es02/es02.key
      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data02:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    networks:
      - elastic

  kib01:
    container_name: kib01
    image: docker.elastic.co/kibana/kibana:7.15.1
    environment:
      SERVER_NAME: kib01
      SERVER_HOST: kib01
      SERVER_PORT: "8080"
      SERVER_SSL_ENABLED: "true"
      SERVER_SSL_CERTIFICATE: $CERTS_DIR/kib01/kib01.crt
      SERVER_SSL_KEY: $CERTS_DIR/kib01/kib01.key
      SERVER_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca/ca.crt
      ELASTICSEARCH_HOSTS: '["https://es01:9243","https://es02:9243"]'
      ELASTICSEARCH_USERNAME: elastic
      ELASTICSEARCH_PASSWORD: $ELASTIC_PASSWORD
      ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca/ca.crt
      TELEMETRY_ENABLED: "false"
    volumes: ['certs:$CERTS_DIR']
    ports:
      - 8080:8080
    networks:
      - elastic

  wait_until_ready:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    command: /usr/bin/true
    depends_on: {"es01": {"condition": "service_healthy"}}

volumes: {"data01", "data02", "certs"}

networks:
  elastic:
    driver: bridge
EOF

Start monitoring stack

$ docker-compose up -d

Test elasticsearch cluster health

$ docker run --rm -v es_certs:/certs --network=es_elastic docker.elastic.co/elasticsearch/elasticsearch:7.15.1 curl --silent --cacert /certs/ca/ca.crt -u elastic:PleaseChangeMe https://es01:9243/_cluster/health

References

https://www.elastic.co/guide/en/elastic-stack-get-started/current/get-started-docker.html

https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-tls-docker.html

https://www.elastic.co/blog/configuring-ssl-tls-and-https-to-secure-elasticsearch-kibana-beats-and-logstash

https://www.elastic.co/guide/en/elasticsearch/reference/current/security-minimal-setup.html#_enable_elasticsearch_security_features

Sample filebeat container configuration

services:
  filebeat:
    container_name: filebeat
    image: docker.elastic.co/beats/filebeat:7.15.1
    user: root
    command: --strict.perms=false -e
    volumes:
      - ./config/filebeat.yaml:/usr/share/filebeat/filebeat.yml:ro
      - /var/log:/mnt/log:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
    healthcheck:
      test: filebeat --strict.perms=false test config
      interval: 30s
      timeout: 15s
      retries: 5
    networks:
      - elastic

Content of config/filebeat.yaml file

setup:
  dashboards.enabled: false
  ilm:
    enabled: false
    check_exists: false
    overwrite: false
  template:
    enabled: false
    overwrite: false
    settings:
      index.number_of_shards: 1
      index.number_of_replicas: 1

queue.mem:
  events: 1024
  flush.min_events: 256
  flush.timeout: 1s

#filebeat.config:
#  modules:
#    path: ${path.config}/modules.d/*.yml
#    reload.enabled: false

filebeat.modules:
- module: system
  syslog:
    enabled: true
    var.paths: ["/mnt/log/syslog*"]

filebeat.autodiscover:
  providers:
    - type: docker
      hints.enabled: true
      #hints.default_config.enabled: false

processors:
- add_docker_metadata:
    host: "unix:///var/run/docker.sock"
- add_host_metadata: ~
- add_cloud_metadata:
    providers: ["aws"]
- add_fields:
    target: "project"
    fields:
      team:
        name: "abc"
      component:
        name: "abc"

logging:
  level: info
  selectors: ["*"]

output.elasticsearch:
  hosts: ["https://es01:9243/"]
  protocol: "https"
  username: "beats_writer"
  password: "PleaseChangeMe"
  ssl.verification_mode: "none"

Configure index as described in:
https://gist.github.com/avoidik/6696edb519215125e5bf044d442ac285

Run it

$ docker-compose -f filebeat.yaml -f docker-compose.yml run --rm filebeat

With enabled hints.default_config.enabled: false we need label containers like so

$ docker run \
  --label co.elastic.logs/module=nginx \
  --label co.elastic.logs/fileset.stdout=access \
  --label co.elastic.logs/fileset.stderr=error \
  --detach=true \
  --name nginx-app \
  -p 8081:80 \
  nginx:alpine

Reference:
https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html
https://www.elastic.co/guide/en/beats/filebeat/current/configuration-autodiscover.html
https://www.elastic.co/guide/en/beats/filebeat/current/configuration-autodiscover-hints.html
https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-template.html

https://www.elastic.co/guide/en/elasticsearch/reference/8.5/index-lifecycle-error-handling.html