Kibana dashboard for the alerting event log

dashboard.ndjson

{"attributes":{"fieldAttrs":"{}","fieldFormatMap":"{\"event.duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"nanoseconds\",\"outputFormat\":\"asMilliseconds\",\"showSuffix\":true,\"useShortSuffix\":true,\"includeSpaceWithSuffix\":true}}}","fields":"[]","runtimeFieldMap":"{\"soid\":{\"type\":\"keyword\",\"script\":{\"source\":\"def savedObjects = params._source[\\\"kibana\\\"][\\\"saved_objects\\\"];\\n\\nif (savedObjects != null) {\\n  for (def savedObject : savedObjects) {\\n    emit(savedObject[\\\"type\\\"] + \\\":\\\" + savedObject[\\\"id\\\"])\\n  }  \\n}\\n\"}}}","timeFieldName":"@timestamp","title":".kibana-event-log-*","typeMeta":"{}"},"coreMigrationVersion":"8.0.0","id":"cd94ad50-c4c0-11eb-a329-07149b9b0aad","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-06-03T23:25:41.302Z","version":"WzY4Nzg3LDFd"}
{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"controlledBy\":\"1622776234798\",\"disabled\":false,\"key\":\"event.provider\",\"negate\":false,\"params\":{\"query\":\"alerting\"},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"event.provider\":\"alerting\"}}}]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"8.0.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":5,\"i\":\"46680c26-7bb5-45ab-b152-cfbb66003155\"},\"panelIndex\":\"46680c26-7bb5-45ab-b152-cfbb66003155\",\"embeddableConfig\":{\"savedVis\":{\"title\":\"\",\"description\":\"\",\"type\":\"input_control_vis\",\"params\":{\"controls\":[{\"id\":\"1622776234798\",\"fieldName\":\"event.provider\",\"parent\":\"\",\"label\":\"\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":false,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_46680c26-7bb5-45ab-b152-cfbb66003155_0_index_pattern\"}],\"updateFiltersOnChange\":true,\"useTimeFilter\":false,\"pinFilters\":false},\"uiState\":{},\"data\":{\"aggs\":[],\"searchSource\":{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"controlledBy\":\"1622776234798\",\"disabled\":false,\"index\":\"cd94ad50-c4c0-11eb-a329-07149b9b0aad\",\"key\":\"event.provider\",\"negate\":false,\"params\":{\"query\":\"alerting\"},\"type\":\"phrase\"},\"query\":{\"match_phrase\":{\"event.provider\":\"alerting\"}}}]}}},\"enhancements\":{}}},{\"version\":\"8.0.0\",\"type\":\"lens\",\"gridData\":{\"x\":24,\"y\":5,\"w\":24,\"h\":11,\"i\":\"78baa5d7-7e3b-488a-96ec-0feee7c613fc\"},\"panelIndex\":\"78baa5d7-7e3b-488a-96ec-0feee7c613fc\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"type\":\"lens\",\"visualizationType\":\"lnsXY\",\"state\":{\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"0cd5ec05-c483-46f6-a6b5-685c8efdbddb\":{\"columns\":{\"b8552648-5daa-4baf-a29d-314b3bec646e\":{\"label\":\"Top values of soid\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"soid\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"column\",\"columnId\":\"8a2a28db-28f4-4fbc-a521-1efecdf6b773\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false}},\"5bb29fc6-ac0d-4a54-92f6-68fa2242278b\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\"}},\"8a2a28db-28f4-4fbc-a521-1efecdf6b773\":{\"label\":\"Count of records\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\"}},\"columnOrder\":[\"b8552648-5daa-4baf-a29d-314b3bec646e\",\"5bb29fc6-ac0d-4a54-92f6-68fa2242278b\",\"8a2a28db-28f4-4fbc-a521-1efecdf6b773\"],\"incompleteColumns\":{}}}}},\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yLeftExtent\":{\"mode\":\"full\"},\"yRightExtent\":{\"mode\":\"full\"},\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"0cd5ec05-c483-46f6-a6b5-685c8efdbddb\",\"accessors\":[\"8a2a28db-28f4-4fbc-a521-1efecdf6b773\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"xAccessor\":\"5bb29fc6-ac0d-4a54-92f6-68fa2242278b\",\"splitAccessor\":\"b8552648-5daa-4baf-a29d-314b3bec646e\",\"palette\":{\"type\":\"palette\",\"name\":\"temperature\"}}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"event.action\",\"params\":{\"query\":\"execute\"},\"indexRefName\":\"filter-index-pattern-0\"},\"query\":{\"match_phrase\":{\"event.action\":\"execute\"}},\"$state\":{\"store\":\"appState\"}}]},\"references\":[{\"type\":\"index-pattern\",\"id\":\"cd94ad50-c4c0-11eb-a329-07149b9b0aad\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"cd94ad50-c4c0-11eb-a329-07149b9b0aad\",\"name\":\"indexpattern-datasource-layer-0cd5ec05-c483-46f6-a6b5-685c8efdbddb\"},{\"name\":\"filter-index-pattern-0\",\"type\":\"index-pattern\",\"id\":\"cd94ad50-c4c0-11eb-a329-07149b9b0aad\"}]},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"event log execution count by SO id\"},{\"version\":\"8.0.0\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":5,\"w\":24,\"h\":11,\"i\":\"e74c7748-1603-4326-b4b4-2d408412a0cd\"},\"panelIndex\":\"e74c7748-1603-4326-b4b4-2d408412a0cd\",\"embeddableConfig\":{\"attributes\":{\"title\":\"event log duration by SO id\",\"type\":\"lens\",\"visualizationType\":\"lnsXY\",\"state\":{\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"6a419c9c-4a61-4a7c-acd1-d7fa0005f517\":{\"columns\":{\"723115a9-e3d5-4bea-b807-a665b41832d8\":{\"label\":\"Top values of soid\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"soid\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"column\",\"columnId\":\"8143411b-70ff-4fa1-953d-b57108c493e1\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false}},\"5910cbee-3009-4c77-9ff5-ae25fa88d981\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\"}},\"8143411b-70ff-4fa1-953d-b57108c493e1\":{\"label\":\"Average of e.duration\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"event.duration\",\"isBucketed\":false,\"scale\":\"ratio\"}},\"columnOrder\":[\"723115a9-e3d5-4bea-b807-a665b41832d8\",\"5910cbee-3009-4c77-9ff5-ae25fa88d981\",\"8143411b-70ff-4fa1-953d-b57108c493e1\"],\"incompleteColumns\":{}}}}},\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"yLeftExtent\":{\"mode\":\"full\"},\"yRightExtent\":{\"mode\":\"full\"},\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"6a419c9c-4a61-4a7c-acd1-d7fa0005f517\",\"accessors\":[\"8143411b-70ff-4fa1-953d-b57108c493e1\"],\"position\":\"top\",\"seriesType\":\"line\",\"showGridlines\":false,\"xAccessor\":\"5910cbee-3009-4c77-9ff5-ae25fa88d981\",\"splitAccessor\":\"723115a9-e3d5-4bea-b807-a665b41832d8\",\"palette\":{\"type\":\"palette\",\"name\":\"status\"}}],\"curveType\":\"CURVE_MONOTONE_X\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]},\"references\":[{\"type\":\"index-pattern\",\"id\":\"cd94ad50-c4c0-11eb-a329-07149b9b0aad\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"cd94ad50-c4c0-11eb-a329-07149b9b0aad\",\"name\":\"indexpattern-datasource-layer-6a419c9c-4a61-4a7c-acd1-d7fa0005f517\"}]},\"enhancements\":{\"dynamicActions\":{\"events\":[{\"eventId\":\"b9a50253-462c-4ed3-b6fe-9239f7046c43\",\"triggers\":[\"VALUE_CLICK_TRIGGER\"],\"action\":{\"factoryId\":\"URL_DRILLDOWN\",\"name\":\"Go to alert\",\"config\":{\"url\":{\"template\":\"{{kibanaUrl}}/app/management/insightsAndAlerting/triggersActions/rule/{{soid}}\"},\"openInNewTab\":true,\"encodeUrl\":true}}}]}}}}]","timeRestore":false,"title":"alerting event log","version":1},"coreMigrationVersion":"8.0.0","id":"e063fcf0-c4c6-11eb-a329-07149b9b0aad","migrationVersion":{"dashboard":"7.14.0"},"references":[{"id":"cd94ad50-c4c0-11eb-a329-07149b9b0aad","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"cd94ad50-c4c0-11eb-a329-07149b9b0aad","name":"46680c26-7bb5-45ab-b152-cfbb66003155:kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"cd94ad50-c4c0-11eb-a329-07149b9b0aad","name":"46680c26-7bb5-45ab-b152-cfbb66003155:control_46680c26-7bb5-45ab-b152-cfbb66003155_0_index_pattern","type":"index-pattern"},{"id":"cd94ad50-c4c0-11eb-a329-07149b9b0aad","name":"78baa5d7-7e3b-488a-96ec-0feee7c613fc:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"cd94ad50-c4c0-11eb-a329-07149b9b0aad","name":"78baa5d7-7e3b-488a-96ec-0feee7c613fc:indexpattern-datasource-layer-0cd5ec05-c483-46f6-a6b5-685c8efdbddb","type":"index-pattern"},{"id":"cd94ad50-c4c0-11eb-a329-07149b9b0aad","name":"78baa5d7-7e3b-488a-96ec-0feee7c613fc:filter-index-pattern-0","type":"index-pattern"},{"id":"cd94ad50-c4c0-11eb-a329-07149b9b0aad","name":"e74c7748-1603-4326-b4b4-2d408412a0cd:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"cd94ad50-c4c0-11eb-a329-07149b9b0aad","name":"e74c7748-1603-4326-b4b4-2d408412a0cd:indexpattern-datasource-layer-6a419c9c-4a61-4a7c-acd1-d7fa0005f517","type":"index-pattern"}],"type":"dashboard","updated_at":"2021-06-04T03:28:36.740Z","version":"WzEzMjgzNCwxXQ=="}
{"exportedCount":2,"missingRefCount":0,"missingReferences":[]}

The dashboard export above adds an index pattern for Kibana event log index.kibana-event-log-* and dashboard that uses it.

Built with a Kibana dev server off master as of 2021-06-03.

The index pattern includes a runtime field soid which will be the type and id of the relevant saved object for the event, extracted from the kibana.saved_objects nested field. For example, alert:1234-....

The dashboard just shows using that field in some Lens graphs, it's not intended to be useful.

If you don't already have some alerts running, here's some code to run using kbn-alert and kbn-action to create an index threshold alert that will generate a lot of events. Delete the action with id 'slack' if you don't have a predefined connector for Slack with the id slack. If you don't, why don't you? :-)

Run the kbn-alert invocation multiple times to get multiple alerts running using the same actions.

export ACTION_ID=`kbn-action create .server-log 'server-log' '{}' '{}' | jq -r '.id'`

kbn-alert create .index-threshold "event log sample " 1s \
  '{
    index:               [".kibana-event-log-*"], 
    timeField:           "@timestamp", 
    aggType:             "count", 
    groupBy:             "top",
    termSize:            10,
    termField:           "event.provider",
    timeWindowSize:      5, 
    timeWindowUnit:      "s", 
    thresholdComparator: ">", 
    threshold:           [0]
  }' \
  "[
    { group: 'threshold met', id: '$ACTION_ID', params: {message: '{{context.message}}'} },
    { group: 'threshold met', id: 'slack',      params: {message: '{{context.message}}'} },
  ]"

runtime field definition for the soid field:

def savedObjects = params._source["kibana"]["saved_objects"];

if (savedObjects != null) {
  for (def savedObject : savedObjects) {
    emit(savedObject["type"] + ":" + savedObject["id"])
  }  
}