index lifecycle management example

docker-compose.yml

version: '2.2'
services:
  kibana:
    image: docker.elastic.co/kibana/kibana:7.6.0
    environment:
      SERVER_NAME: 127.0.0.1
      ELASTICSEARCH_URL: http://elasticsearch:9200/
    ports:
      - 5601:5601
    logging:
      driver: none
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.0
    environment:
      - node.name=elasticsearch
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=elasticsearch,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data01:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.0
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=elasticsearch,es03
      - cluster.initial_master_nodes=elasticsearch,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data02:/usr/share/elasticsearch/data
  es03:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.0
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=elasticsearch,es02
      - cluster.initial_master_nodes=elasticsearch,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data03:/usr/share/elasticsearch/data

volumes:
  data01:
    driver: local
  data02:
    driver: local
  data03:
    driver: local

es-fill-data.go

package main

import (
  "time"
  "fmt"
  "bytes"
  "net/http"
  "encoding/json"
  guuid "github.com/google/uuid"
)

func main() {
  for {
    s := run()
    if s != nil {
      fmt.Println(s)
    }
  }
}

func run() error {
  type Payload struct {
    Timestamp string `json:"@timestamp"`
  	User    string `json:"user"`
  	Message string `json:"message"`
  }
  t := time.Now()
  data := Payload{
    Timestamp: t.Format(time.RFC3339Nano),
    User: guuid.New().String(),
    Message: guuid.New().String(),
  }
  payloadBytes, err := json.Marshal(data)
  if err != nil {
  	return err
  }
  fmt.Println(string(payloadBytes))
  body := bytes.NewReader(payloadBytes)
  
  req, err := http.NewRequest("POST", "http://127.0.0.1:9200/datastream/_doc/?pretty", body)
  if err != nil {
  	return err
  }
  req.Header.Set("Content-Type", "application/json")
  
  resp, err := http.DefaultClient.Do(req)
  if err != nil {
  	return err
  }
  
  resp.Body.Close()
  
  return nil
}

setup.sh

#!/bin/bash -xe

curl -s -X PUT "127.0.0.1:9200/_cluster/settings" -H 'Content-Type: application/json' -d'
{
  "transient": {
    "indices.lifecycle.poll_interval": "10s"
  }
}
' | jq

curl -X PUT "127.0.0.1:9200/_ilm/policy/datastream_policy?pretty" -H 'Content-Type: application/json' -d'
{
  "policy": {
    "phases": {
      "hot": {          
        "min_age": "10s",            
        "actions": {
          "rollover": {
            "max_size": "1MB",     
            "max_age": "90s"
          }
        }
      },
      "delete": {
        "min_age": "120s",           
        "actions": {
          "delete": {}              
        }
      }
    }
  }
}
'

curl -s "127.0.0.1:9200/_ilm/policy/datastream_policy" | jq


curl -X PUT "127.0.0.1:9200/_template/datastream_template?pretty" -H 'Content-Type: application/json' -d'
{
  "index_patterns": ["datastream-*"],                 
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 1,
    "index.lifecycle.name": "datastream_policy",      
    "index.lifecycle.rollover_alias": "datastream"    
  }
}
'



curl -X PUT "127.0.0.1:9200/datastream-000001?pretty" -H 'Content-Type: application/json' -d'
{
  "aliases": {
    "datastream": {
      "is_write_index": true
    }
  }
}
'

curl -X GET "127.0.0.1:9200/datastream-*/_ilm/explain?pretty"