Elastic Bytes - Runtime fields

runtime-fields.json

##################
# Runtime fields #
##################

GET /

DELETE orders

PUT orders
{
  "mappings": {
    "properties": {
      "@timestamp" : { "type" : "date" },
      "items" : { "type" :"nested"}
    }
  }
}

PUT orders/_doc/1
{
  "@timestamp" : "2021-03-11T12:34:56.789Z",
  "total" : 168.43,
  "items" : [
    {
      "name": "Hoodie Foo",
      "price" : 99.00
    },
    {
      "name" : "Jeans",
      "price" : 56.43
    },
    {
      "name" : "shipping express",
      "price" : 11.00
    }
  ]
}


PUT orders/_doc/2
{
  "@timestamp" : "2021-03-15T12:59:56.789Z",
  "total" : 99,
  "items" : [
    {
      "name": "Hoodie Foo",
      "price" : 99.00
    }
  ]
}


PUT orders/_doc/3
{
  "@timestamp" : "2021-03-16T18:59:56.789CET",
  "total" : 56.43,
  "items" : [
    {
      "name": "Jeans",
      "price" : 56.43
    }
  ]
}

# good old aggs
GET orders/_search
{
  "size": 0,
  "aggs": {
    "by_day": {
      "date_histogram": {
        "field": "@timestamp",
        "interval": "day",
        "min_doc_count": 1
      }
    }
  }
}

# how can we figure out the most common oder hour of day
# and also use this in queries as a range filter?

# with scripts in the agg as well as a script filter
# let's try something better and create a runtime field
POST orders/_mapping
{
  "runtime": {
    "hour_of_day": {
      "type": "long",
      "script": {
        "source": "emit(doc['@timestamp'].value.hour)"
      }
    }
  }
}

# let's take a look
GET orders/_search?filter_path=**.hour_of_day,**.@timestamp
{
  "fields" : ["_source", "hour_of_day"]
}

# aggs
GET orders/_search
{
  "size": 0, 
  "aggs": {
    "by_hour": {
      "histogram": {
        "field": "hour_of_day",
        "interval": 1,
        "min_doc_count": 1
      }
    }
  }
}

# search against that field
GET orders/_search
{
  "query": {
    "range": {
      "hour_of_day": {
        "lt": 13
      }
    }
  }
}

# also there is no need to change the mapping
# you can also specify the field in the search request
# let's search for the month instead
GET orders/_search
{
  "runtime_mappings": {
    "month": {
      "type": "long",
      "script": {
        "source": "emit(doc['@timestamp'].value.monthValue)"
      }
    }
  },
  "query": {
    "range": {
      "month": {
        "lt": 4
      }
    }
  }
}


# emitting more than a single field - via grok
DELETE logs 

PUT logs/_doc/1
{
  "message": "2021-01-18T14:03:54.870920Z   INFO - User has logged out -- user_id: 1, user_name: \"Alex\""
}

PUT logs/_doc/2
{
  "message": "2021-01-18T14:24:11.254881Z  ERROR - Error occured"
}

POST logs/_mapping
{
  "runtime": {
    "@timestamp": {
      "type": "date",
      "script": {
        "source": """
         def dateAsString = grok('%{DATA:date} %{GREEDYDATA:input}').extract(doc["message.keyword"].value)?.date;
         emit(Instant.parse(dateAsString).toEpochMilli());
        """
      }
    }
  }
}

# let's take a look
GET logs/_search?filter_path=**.@timestamp
{
  "fields" : ["_source", "@timestamp"]
}

GET logs/_search
{
  "query": {
    "range": {
      "@timestamp": {
        "gte": "2021-01-18"
      }
    }
  }
}


# you may want to use dissect where possible for performance
POST logs/_mapping
{
  "runtime": {
    "@timestamp": {
      "type": "date",
      "script": {
        "source": """
         def dateAsString = dissect('%{date} %{input}').extract(doc["message.keyword"].value)?.date;
         emit(Instant.parse(dateAsString).toEpochMilli());
        """
      }
    }
  }
}

GET logs/_mapping

GET logs/_search
{
  "query": {
    "range": {
      "@timestamp": {
        "gte": "2021-01-18"
      }
    }
  }
}


# production hint: switch to asynchronous search in production
POST logs/_async_search
{
  "query": {
    "range": {
      "@timestamp": {
        "gte": "2021-01-18"
      }
    }
  }
}

# you can also search indices mixing this as a real/runtime field
# changing the mapping in your index template from indexed field to runtime field or vice versa is no problem
DELETE logs-2

PUT logs-2

PUT logs-2/_doc/1
{
  "@timestamp" : "2021-01-18T14:03:54.870920Z",
  "message": "2021-01-18T14:03:54.870920Z   INFO - User has logged out -- user_id: 1, user_name: \"Alex\""
}

PUT logs-2/_doc/2
{
  "@timestamp" : "2021-01-18T14:24:11.254881Z",
  "message": "2021-01-18T14:24:11.254881Z  ERROR - Error occured"
}

GET logs*/_search
{
  "query": {
    "range": {
      "@timestamp": {
        "gte": "2021-01-18"
      }
    }
  }
}

# future, potentially emit several fields at once
# future, runtime field editor at index patterns
# please leave feedback, create issues


# Kahoot: https://kahoot.it/challenge/09238643