Painless Pipeline example to remove and rename fields

181840.txt

# Add a test document to the index

PUT index-2018-08-09/logs_2/AWUbrXRdjZ-987654akh
{
"datetime_log": "2018-08-09T00:34:36.051+02:00",
"datetime_receive": "2018-08-09T00:34:36.051+02:00",
"group": "DEFAUT",
"ip_host": "22.33.44.55",
"ip_host_pkt": "22.33.44.55",
"source_msg": "22.33.44.55: -Trashback= XXXXXXXXX XXXXXXXXXX XXXXXXXXX 8002DAD8 YYYYYYYY ZZZZZZZZZ UUUUUUUU IIIIIIIIII",
"unix_level": "local7",
"unix_priority": "crit"
}

# Create an Ingest Pipeline to rename and remove fields 
PUT _ingest/pipeline/rename_fields
{
  "description": "rename (datetime_receive,ip_host_pkt,source_msg) and renamed them to date,host,message)",
  "processors": [
    {"rename": {"field": "datetime_receive","target_field": "date"}},
    {"rename": {"field": "ip_host_pkt","target_field": "host"}},
    {"rename": {"field": "source_msg","target_field": "message"}},
    {"remove": {"field": "unix_priority"}}
    ]
}

# Test the pipeline before making any changes to the indices
POST _ingest/pipeline/rename_fields/_simulate
{
  "docs" : [
    { "_source": {
"datetime_log": "2018-08-09T00:34:36.051+02:00",
"datetime_receive": "2018-08-09T00:34:36.051+02:00",
"group": "DEFAUT",
"ip_host": "22.33.44.55",
"ip_host_pkt": "22.33.44.55",
"source_msg": "22.33.44.55: -Trashback= XXXXXXXXX XXXXXXXXXX XXXXXXXXX 8002DAD8 YYYYYYYY ZZZZZZZZZ UUUUUUUU IIIIIIIIII",
"unix_level": "local7",
"unix_priority": "crit"
} }
  ]
}

# Expected result:
{
  "docs": [
    {
      "doc": {
        "_index": "_index",
        "_type": "_type",
        "_id": "_id",
        "_source": {
          "unix_level": "local7",
          "date": "2018-08-09T00:34:36.051+02:00",
          "datetime_log": "2018-08-09T00:34:36.051+02:00",
          "ip_host": "22.33.44.55",
          "host": "22.33.44.55",
          "message": "22.33.44.55: -Trashback= XXXXXXXXX XXXXXXXXXX XXXXXXXXX 8002DAD8 YYYYYYYY ZZZZZZZZZ UUUUUUUU IIIIIIIIII",
          "group": "DEFAUT"
        },
        "_ingest": {
          "timestamp": "2018-10-23T12:52:42.444138Z"
        }
      }
    }
  ]
}

# Send all the documents in the index through the pipleine to a new index
POST _reindex
{
  "source": {
    "index": "index-2018-08-09"
  },
  "dest": {
    "index": "index-2018-08-09-new",
    "pipeline": "rename_fields"
  }
}

# check the result
GET index-2018-08-09-new/logs_2/AWUbrXRdjZ-987654akh

# returned document is:
{
  "_index": "index-2018-08-09-new",
  "_type": "logs_2",
  "_id": "AWUbrXRdjZ-987654akh",
  "_version": 2,
  "found": true,
  "_source": {
    "date": "2018-08-09T00:34:36.051+02:00",
    "datetime_log": "2018-08-09T00:34:36.051+02:00",
    "message": "22.33.44.55: -Trashback= XXXXXXXXX XXXXXXXXXX XXXXXXXXX 8002DAD8 YYYYYYYY ZZZZZZZZZ UUUUUUUU IIIIIIIIII",
    "unix_level": "local7",
    "ip_host": "22.33.44.55",
    "host": "22.33.44.55",
    "group": "DEFAUT"
  }
}

# You can also update the document IN THE ORIGINAL INDEX (USE WITH EXTRA CARE) using the "_update_by_query" endpoint

# This request will apply the pipeline to ALL documents in the original index
POST index-2018-08-09/_update_by_query?pipeline=rename_fields

# check the result
GET index-2018-08-09/logs_2/AWUbrXRdjZ-987654akh