Andrew Kroh andrewkroh

Adding event.ingested and lag calculations to Winlogbeat events

Create an Ingest Pipeline that will add four fields:

event.ingested - Time when the event was processed by Elasticsearch.
event.lag.read - Time difference in milliseconds between @timestamp and event.created. This measures how long it took for Winlogbeat read the event from the event log (for WEC this includes the delivery time from forwarder to collector).
event.lag.ingest - Time difference in milliseconds between event.created and event.ingested. This measures the time between Winlogbeat reading the event (time when it "created" the document) to when it was written to Elasticsearch.

	#!/usr/bin/env bash
	# Licensed to Elasticsearch B.V. under one or more contributor
	# license agreements. See the NOTICE file distributed with
	# this work for additional information regarding copyright
	# ownership. Elasticsearch B.V. licenses this file to you under
	# the Apache License, Version 2.0 (the "License"); you may
	# not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http:#www.apache.org/licenses/LICENSE-2.0

	<!-- GE(Jasco) 46203 Z-Wave Plus Dimmer Switch -->
	<!-- Configuration Parameters - per http://products.z-wavealliance.org/products/3323 -->
	<Product Revision="1" xmlns="https://github.com/OpenZWave/open-zwave">
	<MetaData>
	<MetaDataItem name="OzwInfoPage">http://www.openzwave.com/device-database/0063:3235:4944</MetaDataItem>
	<MetaDataItem name="ProductPic">images/ge/46203-dimmer.png</MetaDataItem>
	<MetaDataItem id="3235" name="ZWProductPage" type="4944">https://products.z-wavealliance.org/products/3323/</MetaDataItem>
	<MetaDataItem name="Name">In-Wall Smart Dimmer </MetaDataItem>
	<MetaDataItem name="ProductManual">https://products.z-wavealliance.org/ProductManual/File?folder=&filename=MarketCertificationFiles/3323/14294.46203.ZW3010%20Binder.pdf</MetaDataItem>
	<MetaDataItem id="3235" name="FrequencyName" type="4944">U.S. / Canada / Mexico</MetaDataItem>

	{
	"description": "Pipeline for parsing Symantec Endpoint logs",
	"processors": [
	{
	"set": {
	"field": "event.original",
	"value": "{{{message}}}"
	}
	},
	{

	{
	"description": "Pipeline for parsing Citrix Netscaler logs",
	"processors": [
	{
	"script": {
	"description": "set event.original",
	"lang": "painless",
	"source": "def event = ctx.event;\nif (event == null) {\n event = [:];\n ctx['event'] = event;\n}\nevent['original'] = ctx.message;\n"
	}
	},

	# --Shell-script--
	#
	# functions This file contains functions to be used by most or all
	# shell scripts in the /etc/init.d directory.
	#

	TEXTDOMAIN=initscripts

	# Make sure umask is sane
	umask 022

	1348870236.160 0 192.168.0.35 TCP_DENIED/403 3293 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
	1348870236.273 0 192.168.0.35 TCP_DENIED/403 3274 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
	1348870236.386 0 192.168.0.35 TCP_DENIED/403 3274 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
	1348870236.499 0 192.168.0.35 TCP_DENIED/403 3274 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.5.2/misc/AcrobatUpd952_all_incr.msp - NONE/- text/html
	1348870237.550 0 192.168.0.35 TCP_DENIED/403 3269 GET http://armdl.adobe.com/pub/adobe/acrobat/win/9.x/9.4.6/misc/AcrobatUpd946_all_incr.msp - NONE/- text/html
	1348870274.248 59875 192.168.0.35 TCP_MISS/503 0 CONNECT client84.dropbox.com:443 - DIRECT/- -
	1348870284.249 59872 192.168.0.35 TCP_MISS/503 0 CONNECT client62.dropbox.com:443 - DIRECT/- -
	1348870

	PUT /_ingest/pipeline/domain_rank_enrichment
	{
	"description" : "Enriching domains with rank.",
	"processors" : [
	{
	"enrich" : {
	"policy_name": "dns-domain-top1m-rank",
	"field" : "dns.question.name",
	"target_field": "_temp"
	}