Skip to content

Instantly share code, notes, and snippets.

@lukas-vlcek

lukas-vlcek/com.redhat.viaq-openshift-project.template.json

Created May 19, 2021 09:41
Show Gist options
  • Save lukas-vlcek/1382f6d46cce7c0d4030fb13cbc674a1 to your computer and use it in GitHub Desktop.
Save lukas-vlcek/1382f6d46cce7c0d4030fb13cbc674a1 to your computer and use it in GitHub Desktop.
Download ZIP
Setup of Openshift schema pipeline
Raw
com.redhat.viaq-openshift-project.template.json
{
"aliases": {
".all": {
}
},
"index_patterns": [
"app*"
],
"mappings": {
"_doc": {
"_meta": {
"version": "2020.01.23.0"
},
"date_detection": false,
"dynamic_templates": [
{
"aushape_generic_nested_fields": {
"mapping": {
"index": true,
"type": "text"
},
"path_match": "aushape.data.*.*.*"
}
},
{
"aushape_generic_fields": {
"mapping": {
"index": true,
"type": "text"
},
"path_match": "aushape.data.*.*"
}
},
{
"aushape_generic_records": {
"mapping": {
"type": "object"
},
"path_match": "aushape.data.*"
}
},
{
"message_field": {
"mapping": {
"index": true,
"norms": false,
"type": "text"
},
"match": "message",
"match_mapping_type": "string"
}
},
{
"string_fields": {
"mapping": {
"fields": {
"raw": {
"ignore_above": 256,
"index": false,
"type": "keyword"
}
},
"index": true,
"norms": true,
"type": "text"
},
"match": "*",
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"doc_values": true,
"fields": {
"raw": {
"doc_values": true,
"ignore_above": 256,
"index": true,
"type": "keyword"
}
},
"format": "yyyy-MM-dd HH:mm:ss,SSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime",
"index": true,
"type": "date"
},
"aushape": {
"properties": {
"data": {
"properties": {
"avc": {
"type": "nested"
},
"execve": {
"doc_values": false,
"index": true,
"type": "text"
},
"netfilter_cfg": {
"type": "nested"
},
"obj_pid": {
"type": "nested"
},
"path": {
"type": "nested"
}
}
},
"error": {
"doc_values": false,
"index": true,
"type": "text"
},
"node": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"serial": {
"doc_values": true,
"index": true,
"type": "long"
},
"text": {
"doc_values": false,
"index": true,
"type": "text"
},
"trimmed": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
},
"docker": {
"properties": {
"command": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"container_id": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"container_id_short": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"container_image": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"container_name": {
"doc_values": false,
"fields": {
"raw": {
"doc_values": true,
"ignore_above": 256,
"index": true,
"type": "keyword"
}
},
"index": true,
"norms": true,
"type": "text"
},
"operation": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"pid": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"reason": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"result": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"sauid": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"user": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
},
"file": {
"doc_values": false,
"fields": {
"raw": {
"doc_values": true,
"ignore_above": 256,
"index": true,
"type": "keyword"
}
},
"index": true,
"norms": true,
"type": "text"
},
"geoip": {
"dynamic": true,
"properties": {
"location": {
"type": "geo_point"
}
},
"type": "object"
},
"hostname": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"ipaddr4": {
"doc_values": true,
"fields": {
"raw": {
"doc_values": true,
"ignore_above": 256,
"index": true,
"type": "keyword"
}
},
"index": true,
"type": "ip"
},
"ipaddr6": {
"doc_values": true,
"index": true,
"type": "ip"
},
"kubernetes": {
"properties": {
"container_name": {
"doc_values": false,
"fields": {
"raw": {
"doc_values": true,
"ignore_above": 256,
"index": true,
"type": "keyword"
}
},
"index": true,
"norms": true,
"type": "text"
},
"event": {
"properties": {
"count": {
"doc_values": true,
"index": true,
"type": "integer"
},
"firstTimestamp": {
"doc_values": true,
"format": "yyyy-MM-dd HH:mm:ss,SSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime",
"index": true,
"type": "date"
},
"involvedObject": {
"properties": {
"apiVersion": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"kind": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"name": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"namespace": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"resourceVersion": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"uid": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
},
"metadata": {
"properties": {
"name": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"namespace": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"resourceVersion": {
"doc_values": true,
"index": true,
"type": "integer"
},
"selfLink": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"uid": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
},
"reason": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"source_component": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"type": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"verb": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
},
"host": {
"doc_values": true,
"index": true,
"norms": true,
"type": "keyword"
},
"labels": {
"properties": {
"component": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"deployment": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"deploymentconfig": {
"doc_values": true,
"fields": {
"raw": {
"doc_values": true,
"ignore_above": 64,
"index": true,
"type": "keyword"
}
},
"index": true,
"type": "keyword"
},
"provider": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
},
"master_url": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"namespace_id": {
"doc_values": true,
"index": true,
"norms": true,
"type": "keyword"
},
"namespace_name": {
"doc_values": true,
"index": true,
"norms": true,
"type": "keyword"
},
"pod_id": {
"doc_values": true,
"index": true,
"norms": true,
"type": "keyword"
},
"pod_name": {
"doc_values": true,
"index": true,
"norms": true,
"type": "keyword"
}
}
},
"level": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"message": {
"doc_values": false,
"index": true,
"norms": false,
"type": "text"
},
"namespace_name": {
"doc_values": false,
"index": true,
"type": "keyword"
},
"namespace_uuid": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"offset": {
"doc_values": true,
"index": true,
"type": "long"
},
"ovirt": {
"properties": {
"class": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"cluster_name": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"correlationid": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"engine_fqdn": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"entity": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"host_id": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"module_lineno": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"thread": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
},
"pid": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"pipeline_metadata": {
"properties": {
"@version": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"collector": {
"properties": {
"hostname": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"inputname": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"ipaddr4": {
"doc_values": true,
"fields": {
"raw": {
"doc_values": true,
"ignore_above": 256,
"index": true,
"type": "keyword"
}
},
"index": true,
"type": "ip"
},
"ipaddr6": {
"doc_values": true,
"index": true,
"type": "ip"
},
"name": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"original_raw_message": {
"doc_values": false,
"fields": {
"raw": {
"doc_values": true,
"ignore_above": 256,
"index": true,
"type": "keyword"
}
},
"index": true,
"type": "text"
},
"received_at": {
"doc_values": true,
"format": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime",
"index": true,
"type": "date"
},
"version": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
},
"normalizer": {
"properties": {
"hostname": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"inputname": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"ipaddr4": {
"doc_values": true,
"fields": {
"raw": {
"doc_values": true,
"ignore_above": 256,
"index": true,
"type": "keyword"
}
},
"index": true,
"type": "ip"
},
"ipaddr6": {
"doc_values": true,
"index": true,
"type": "ip"
},
"name": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"original_raw_message": {
"doc_values": false,
"fields": {
"raw": {
"doc_values": true,
"ignore_above": 256,
"index": true,
"type": "keyword"
}
},
"index": true,
"type": "text"
},
"received_at": {
"doc_values": true,
"format": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime",
"index": true,
"type": "date"
},
"version": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
},
"trace": {
"analyzer": "whitespace",
"doc_values": false,
"index": true,
"type": "text"
}
}
},
"rsyslog": {
"properties": {
"appname": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"facility": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"msgid": {
"doc_values": true,
"index": true,
"type": "long"
},
"protocol-version": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"structured-data": {
"doc_values": true,
"index": true,
"norms": false,
"type": "keyword"
}
}
},
"service": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"systemd": {
"properties": {
"k": {
"properties": {
"KERNEL_DEVICE": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"KERNEL_SUBSYSTEM": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"UDEV_DEVLINK": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"UDEV_DEVNODE": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"UDEV_SYSNAME": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
},
"t": {
"properties": {
"AUDIT_LOGINUID": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"AUDIT_SESSION": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"BOOT_ID": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"CAP_EFFECTIVE": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"CMDLINE": {
"doc_values": true,
"index": true,
"norms": false,
"type": "keyword"
},
"COMM": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"EXE": {
"doc_values": true,
"index": true,
"norms": false,
"type": "keyword"
},
"GID": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"HOSTNAME": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"LINE_BREAK": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"MACHINE_ID": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"PID": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"SELINUX_CONTEXT": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"SOURCE_REALTIME_TIMESTAMP": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"STREAM_ID": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"SYSTEMD_CGROUP": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"SYSTEMD_INVOCATION_ID": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"SYSTEMD_OWNER_UID": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"SYSTEMD_SESSION": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"SYSTEMD_SLICE": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"SYSTEMD_UNIT": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"SYSTEMD_USER_UNIT": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"TRANSPORT": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"UID": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
},
"u": {
"properties": {
"CODE_FILE": {
"doc_values": true,
"index": true,
"norms": false,
"type": "keyword"
},
"CODE_FUNCTION": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"CODE_LINE": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"ERRNO": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"MESSAGE_ID": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"RESULT": {
"doc_values": true,
"index": true,
"norms": false,
"type": "keyword"
},
"SYSLOG_FACILITY": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"SYSLOG_IDENTIFIER": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"SYSLOG_PID": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"UNIT": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
}
}
},
"tags": {
"analyzer": "whitespace",
"doc_values": false,
"index": true,
"type": "text"
},
"tlog": {
"properties": {
"id": {
"doc_values": true,
"index": true,
"type": "long"
},
"in_bin": {
"doc_values": true,
"index": true,
"type": "short"
},
"in_txt": {
"doc_values": false,
"index": true,
"type": "text"
},
"out_bin": {
"doc_values": true,
"index": true,
"type": "short"
},
"out_txt": {
"doc_values": false,
"index": true,
"type": "text"
},
"pos": {
"doc_values": true,
"index": true,
"type": "long"
},
"session": {
"doc_values": true,
"index": true,
"type": "long"
},
"term": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"timing": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"user": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"ver": {
"doc_values": true,
"index": true,
"type": "long"
}
}
},
"viaq_index_name": {
"doc_values": true,
"index": true,
"type": "keyword"
},
"viaq_msg_id": {
"doc_values": true,
"index": true,
"type": "keyword"
}
}
}
},
"order": 10,
"settings": {
"index.refresh_interval": "5s",
"index.default_pipeline": "openshift_schema"
}
}
Raw
openshift_schema_pipeline.json
{
"description": "Openshift inget pipeline for custom schema",
"processors": [
{ "openshift-ingestion-processor": {} }
]
}
@lukas-vlcek
Copy link
Author

lukas-vlcek commented May 19, 2021

Define openshift_schema pipeline in ES cluster:

DOC=`cat ./openshift_schema_pipeline.json`; \
oc exec <es_pod> -c elasticsearch -- es_util --query="_ingest/pipeline/openshift_schema" -X PUT \
-d "$DOC"

Update index template for all app-* indices to execute the openshift_schema pipeline as a default option on indexed documents. All it takes is adding just one line in the end of the template:

DOC=`cat ./com.redhat.viaq-openshift-project.template.json`; \
oc exec <es_pod> -c elasticsearch -- es_util --query="_template/pipeline/com.redhat.viaq-openshift-project.template.json" -X PUT \
-d "$DOC"

@lukas-vlcek
Copy link
Author

lukas-vlcek commented May 19, 2021

EDIT: I added all the above changes to openshift/origin-aggregated-logging#2122

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment