Minimal Elastic monitoring stack

You may use t3.medium AWS EC2 instance to run this stack.

Install docker runtime

$ sudo curl -fsSL https://get.docker.com -o get-docker.sh
$ sudo sh get-docker.sh
$ sudo usermod -a -G docker $USER

Install docker-compose binary

$ sudo curl -fsSL "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
$ sudo chmod +x /usr/local/bin/docker-compose

Increase allowed virtual memory

$ sudo sysctl -w vm.max_map_count=262144

Prepare directory

$ mkdir monitoring
$ cd monitoring

Prepare environment configuration for docker-compose

$ cat <<'EOF' > .env
COMPOSE_PROJECT_NAME=es 
CERTS_DIR=/usr/share/elasticsearch/config/certificates 
ELASTIC_PASSWORD=PleaseChangeMe
EOF

Replace PleaseChangeMe with something else

Prepare list of instances

$ cat <<'EOF' > instances.yml
instances:
- name: es01
  dns:
  - es01 
  - localhost
  ip:
  - 127.0.0.1
- name: es02
  dns:
  - es02
  - localhost
  ip:
  - 127.0.0.1
- name: kib01
  dns:
  - kib01
  - localhost
  ip:
  - 127.0.0.1
EOF

We will use this file to generate x509 certificates

Generate certificates

Prepare docker-compose file

$ cat <<'EOF' > create-certs.yml
version: '2.2'

services:
  create_certs:
    container_name: create_certs
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    command: >
      bash -c '
        if [[ ! -f /certs/bundle.zip ]]; then
          bin/elasticsearch-certutil cert --silent --pem --in config/certificates/instances.yml -out /certs/bundle.zip;
          unzip /certs/bundle.zip -d /certs; 
        fi;
        chown -R 1000:0 /certs
      '
    user: "0"
    working_dir: /usr/share/elasticsearch
    volumes: ['certs:/certs', '.:$CERTS_DIR']

volumes: {"certs"}
EOF

Generate certificates

$ docker-compose -f create-certs.yml run --rm create_certs

Run Elastic and Kibana

Prepare docker-compose file

$ cat <<'EOF' > docker-compose.yml
version: '2.2'

services:
  es01:
    container_name: es01
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    environment:
      - node.name=es01
      - http.port=9243
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - xpack.license.self_generated.type=trial 
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es01/es01.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate 
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key
      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data01:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    ports:
      - 9243:9243
    healthcheck:
      test: curl --silent --cacert $CERTS_DIR/ca/ca.crt https://localhost:9243 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - elastic

  es02:
    container_name: es02
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    environment:
      - node.name=es02
      - http.port=9243
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - xpack.license.self_generated.type=trial
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es02/es02.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate 
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es02/es02.key
      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data02:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    networks:
      - elastic

  kib01:
    container_name: kib01
    image: docker.elastic.co/kibana/kibana:7.15.1
    environment:
      SERVER_NAME: kib01
      SERVER_HOST: kib01
      SERVER_PORT: "8080"
      SERVER_SSL_ENABLED: "true"
      SERVER_SSL_CERTIFICATE: $CERTS_DIR/kib01/kib01.crt
      SERVER_SSL_KEY: $CERTS_DIR/kib01/kib01.key
      SERVER_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca/ca.crt
      ELASTICSEARCH_HOSTS: '["https://es01:9243","https://es02:9243"]'
      ELASTICSEARCH_USERNAME: elastic
      ELASTICSEARCH_PASSWORD: $ELASTIC_PASSWORD
      ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca/ca.crt
      TELEMETRY_ENABLED: "false"
    volumes: ['certs:$CERTS_DIR']
    ports:
      - 8080:8080
    networks:
      - elastic

  wait_until_ready:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    command: /usr/bin/true
    depends_on: {"es01": {"condition": "service_healthy"}}

volumes: {"data01", "data02", "certs"}

networks:
  elastic:
    driver: bridge
EOF

Start monitoring stack

$ docker-compose up -d

Test elasticsearch cluster health

$ docker run --rm -v es_certs:/certs --network=es_elastic docker.elastic.co/elasticsearch/elasticsearch:7.15.1 curl --silent --cacert /certs/ca/ca.crt -u elastic:PleaseChangeMe https://es01:9243/_cluster/health