ingest node example

ingest node example

DELETE _all

PUT _ingest/pipeline/rename_hostname
{
  "processors": [
    {
      "rename": {
        "field": "hostname",
        "target_field": "host",
        "ignore_missing": true
      }
    }
  ]
}

PUT foo/bar/1
{
  "hostname" : "host-001.example.org"
}

GET foo/bar/1

PUT foo/bar/1?pipeline=rename_hostname
{
  "hostname" : "host-001.example.org"
}

GET _ingest/pipeline/rename_hostname

DELETE _ingest/pipeline/rename_hostname

POST _ingest/pipeline/_simulate
{
  "pipeline": {
    "description": "Ingest pipeline for Combined Log Format",
    "processors": [
      {
        "grok": {
          "field": "message",
          "patterns": [
            "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}"
          ]
        }
      },
      {
        "date": {
          "field": "timestamp",
          "formats": [
            "dd/MMM/YYYY:HH:mm:ss Z"
          ]
        }
      },
      {
        "geoip": {
          "field": "clientip"
        }
      },
      {
        "user_agent": {
          "field": "agent"
        }
      }
    ]
  },
  "docs": [
    {
      "_source": {
        "message": "212.87.37.154 - - [12/Sep/2016:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\""
      }
    }
  ]
}

# Script processor
POST _ingest/pipeline/_simulate
{
  "pipeline": {
    "processors": [
      {
        "script": {
          "inline": "ctx.bytes_total = ctx.bytes_in + ctx.bytes_out"
          
        }
      }
    ]
  },
  "docs": [
    {
      "_source": {
        "bytes_in": 1234,
        "bytes_out": 4321
      }
    }
  ]
}


# Foreach processor
# Don't forget to show _ingest field
POST _ingest/pipeline/_simulate
{
  "pipeline": {
    "processors": [
      {
        "foreach": {
          "field" : "values",
          "processor" : {
            "convert" : {
              "field" : "_ingest._value.id",
              "type" : "integer"
            }
          }
        }
      }
    ]
  },
  "docs": [
    {
      "_source": {
        "values": [
          {"name": "first", "id": "1"  },
          {"name": "second", "id": "2" },
          {"name": "third", "id": "3"  }
          ]
      }
    }
  ]
}


# Setting metadata
POST _ingest/pipeline/_simulate
{
  "pipeline": {
    "processors": [
      {
        "set": {
          "field": "_id",
          "value": "foo"
        }
      },
      {
        "date_index_name": {
          "field": "_ingest.timestamp",
          "index_name_prefix": "whatever-",
          "date_rounding": "d"
        }
      }
    ]
  },
  "docs": [
    {
      "_source": {
        "foo": "bar"
      }
    }
  ]
}


# Handling failures, dead letter queue
POST _ingest/pipeline/_simulate
{
  "pipeline": {
    "description": "Ingest pipeline for Combined Log Format",
    "on_failure": [
      {
        "set": {
          "field": "_index",
          "value": "failed-{{ _index }}"
        }
      }
    ],
    "processors": [
      {
        "convert": {
          "field": "non-existing",
          "type": "integer"
        }
      }
    ]
  },
  "docs": [
    {
      "_index": "products",
      "_source": {
        "foo": "bar"
      }
    }
  ]
}



# Get stats
GET /_nodes/stats/ingest