A quick memo on setting up ILM and Datastreams in Elasticsearch

ilm_vs_ds.md

# Comparing ILM to datastreams in terms of set up

# ===================================
# Clean up and settings to help with demo

GET _cat/indices/exp_*
GET _data_stream/exp_ds*
GET _ilm/policy/exp_policy
GET _index_template/exp_temp
GET _index_template/exp_dstemp

PUT _cluster/settings
{
  "transient": {
    "indices.lifecycle.poll_interval": null
  }
}

PUT _cluster/settings
{
  "transient": {
    "indices.lifecycle.poll_interval": "5s"
  }
}


DELETE exp_*
DELETE _data_stream/exp_ds*
DELETE _ilm/policy/exp_policy
DELETE _ilm/policy/exp_dspolicy
DELETE _index_template/exp_temp
DELETE _index_template/exp_dstemp

# ===================================
# Create an index to act as the non-ilm managed index

POST exp_oldindex/_bulk
{"index": {}}
{"foo": "bar", "val": 1, "@timestamp": "2021-04-12T00:00:00.000Z"}
{"index": {}}
{"foo": "bar", "val": 2, "@timestamp": "2021-04-13T00:00:00.000Z"}
{"index": {}}
{"foo": "bar", "val": 3, "@timestamp": "2021-04-14T00:00:00.000Z"}
{"index": {}}
{"foo": "bar", "val": 4, "@timestamp": "2021-04-15T00:00:00.000Z"}
{"index": {}}
{"foo": "bar", "val": 5, "@timestamp": "2021-04-16T00:00:00.000Z"}
{"index": {}}
{"foo": "bar", "val": 6, "@timestamp": "2021-04-17T00:00:00.000Z"}
{"index": {}}
{"foo": "bar", "val": 7, "@timestamp": "2021-04-18T00:00:00.000Z"}

GET _cat/indices/exp_*?v
GET _cat/shards/exp_*?v

# ===================================
# Setting up ILM for normal indices

PUT _ilm/policy/exp_policy
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_age": "30d",
            "max_size": "50gb",
            "max_docs": 3
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "warm": {
        "min_age": "5s",
        "actions": {
          "set_priority": {
            "priority": 50
          },
          "allocate": {
            "number_of_replicas": 0
          }
        }
      }
    }
  }
}

GET _ilm/policy/exp_policy

PUT _index_template/exp_temp
{
  "template": {
    "settings": {
      "index.lifecycle.name": "exp_policy",
      "index.lifecycle.rollover_alias": "exp_alias"
    }
  },
  "index_patterns": [
    "exp_index*"
  ]
}

GET _index_template/exp_temp

PUT exp_index-000001
{
  "aliases": {
    "exp_alias":{
      "is_write_index": true 
    }
  }
}

POST _reindex
{
  "source": {
    "index": "exp_oldindex"
  },
  "dest": {
    "index": "exp_alias"
  }
}

GET _cat/shards/exp_*
GET _cat/indices/exp_*?v

GET exp_index*
GET exp_index*/_ilm/explain

POST exp_alias/_bulk
{"index": {}}
{"foo": "bar", "val": 8, "@timestamp": "2021-04-19T00:00:00.000Z"}
{"index": {}}
{"foo": "bar", "val": 9, "@timestamp": "2021-04-20T00:00:00.000Z"}
{"index": {}}
{"foo": "bar", "val": 10, "@timestamp": "2021-04-21T00:00:00.000Z"}
{"index": {}}
{"foo": "bar", "val": 11, "@timestamp": "2021-04-22T00:00:00.000Z"}

GET exp_alias/_search

GET _cat/shards/exp_*
GET _cat/indices/exp_*?v

# =================================
# Set up a Datastream.

PUT _ilm/policy/exp_dspolicy
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_age": "30d",
            "max_size": "50gb",
            "max_docs": 3
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "warm": {
        "min_age": "5s",
        "actions": {
          "set_priority": {
            "priority": 50
          },
          "allocate": {
            "number_of_replicas": 0
          }
        }
      }
    }
  }
}

PUT _index_template/exp_dstemp
{
  "template": {
    "settings": {
      "index.lifecycle.name": "exp_dspolicy"
    }
  },
  "data_stream": {},
  "index_patterns": [
    "exp_ds*"
  ]
}

PUT _data_stream/exp_ds

POST _reindex
{
  "source": {
    "index": "exp_oldindex"
  },
  "dest": {
    "index": "exp_ds",
    "op_type": "create"                             
  }
}

GET exp_ds/_search

GET _cat/shards/exp_ds*
GET _cat/indices/exp_ds*?v

POST exp_ds/_bulk
{"create": {}}
{"foo": "bar", "val": 8, "@timestamp": "2021-04-19T00:00:00.000Z"}
{"create": {}}
{"foo": "bar", "val": 9, "@timestamp": "2021-04-20T00:00:00.000Z"}
{"create": {}}
{"foo": "bar", "val": 10, "@timestamp": "2021-04-21T00:00:00.000Z"}
{"create": {}}
{"foo": "bar", "val": 11, "@timestamp": "2021-04-22T00:00:00.000Z"}

GET exp_ds/_search

GET _cat/shards/exp_ds*
GET _cat/indices/exp_ds*?v


GET _cat/indices/exp_*
GET _data_stream/exp_*