Created
February 8, 2016 18:22
-
-
Save pblittle/d0b72b5c3cc59fd61077 to your computer and use it in GitHub Desktop.
This is the manifest from https://github.com/logsearch/logsearch-for-cloudfoundry/pull/85
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| jobs: | |
| - instances: 1 | |
| name: ingestor_cf | |
| networks: | |
| - name: default | |
| resource_pool: ingestor | |
| templates: | |
| - name: ingestor_cloudfoundry-firehose | |
| release: logsearch-for-cloudfoundry | |
| properties: | |
| cloudfoundry: | |
| firehose_password: admin | |
| firehose_port: 443 | |
| firehose_user: admin | |
| skip_ssl_validation: true | |
| system_domain: 10.244.0.34.xip.io | |
| elasticsearch_config: | |
| templates: | |
| - "{\n \"template\" : \"logs-*\",\n \"order\": 50,\n \"settings\" : {\n \"index\" | |
| : {\n \"codec\": \"best_compression\",\n \"search\" : {\n \"slowlog\" | |
| : {\n \"threshold\" : {\n \"query\" : {\n \"warn\" | |
| : \"30s\",\n \"info\" : \"15s\",\n \"debug\" : \"10s\",\n | |
| \ \"trace\" : \"5s\"\n }\n }\n }\n },\n | |
| \ \"query\" : { \"default_field\" : \"@message\" }\n }\n },\n \"mappings\" | |
| : {\n \"_default_\" : {\n \"_all\" : {\"enabled\" : false},\n \"dynamic_templates\" | |
| : [ {\n \"message_field\" : {\n \"match\" : \"message\",\n | |
| \ \"match_mapping_type\" : \"string\",\n \"mapping\" : {\n | |
| \ \"type\" : \"string\", \"index\" : \"analyzed\", \"omit_norms\" | |
| : true\n }\n }\n }, {\n \"string_fields\" : | |
| {\n \"match\" : \"*\",\n \"match_mapping_type\" : \"string\",\n | |
| \ \"mapping\" : {\n \"type\" : \"string\", \n \"index\" | |
| : \"not_analyzed\", \n \"omit_norms\" : true\n }\n }\n | |
| \ } ],\n \"properties\" : {\n \"@version\": { \"type\": \"string\", | |
| \"index\": \"not_analyzed\" },\n \"@message\" : {\n \"type\" | |
| : \"string\",\n \"index\" : \"analyzed\",\n \"norms\" | |
| : { \"enabled\" : false }\n },\n \"geoip\" : {\n \"type\" | |
| : \"object\",\n \"dynamic\": true,\n \"properties\" | |
| : {\n \"location\" : { \"type\" : \"geo_point\" }\n }\n | |
| \ }\n }\n }\n }\n}\n" | |
| logstash_parser: | |
| filters: "# All logs start being sent to the unparsed index. \n# The filters | |
| below will route them to the @index=app or @index=platform\nif ! [@metadata][index] | |
| {\n mutate {\n add_field => { \"[@metadata][index]\" => \"unparsed\" | |
| }\n add_field => { \"[@metadata][type]\" => \"%{[@type]}\" }\n }\n}\n\nif | |
| [@metadata][type] in [\"syslog\", \"relp\"] and [syslog_program] == \"doppler\" | |
| {\n# Parse Cloud Foundry logs from doppler firehose (via https://github.com/SpringerPE/firehose-to-syslog)\n\njson | |
| {\n source => 'syslog_message'\n}\n\nif \"_jsonparsefailure\" in [tags] {\n\n | |
| \ # Amend the failure tag to match our fail/${addon}/${filter}/${detail} standard\n | |
| \ mutate {\n add_tag => [\"fail/cloudfoundry/firehose/jsonparsefailure_of_syslog_message\"]\n | |
| \ remove_tag => [\"_jsonparsefailure\"]\n }\n\n} else {\n\n date | |
| {\n match => [ \"time\", \"ISO8601\" ]\n }\n\n #Ensure that we | |
| always have an event_type, in prep for adding metrics\n if ![event_type] | |
| {\n mutate {\n add_field => [ \"event_type\", \"LogMessage\" | |
| ]\n }\n }\n\n if [event_type] == \"ContainerMetric\" {\n mutate | |
| {\n add_tag => \"ContainerMetric\"\n add_field => {\"[@source][component]\" | |
| => \"METRIC\"}\n add_field => {\"[@source][instance]\" => \"%{[instance_index]}\"}\n | |
| \ }\n\n mutate {\n rename => { \"[cpu_percentage]\" => \"[container][cpu_percentage]\" | |
| }\n rename => { \"[memory_bytes]\" => \"[container][memory_bytes]\" }\n | |
| \ rename => { \"[disk_bytes]\" => \"[container][disk_bytes]\" }\n remove | |
| => \"[instance_index]\"\n }\n }\n\n\n\n mutate {\n rename | |
| => { \"[cf_app_id]\" => \"[@source][app][id]\" }\n rename => { \"[cf_app_name]\" | |
| => \"[@source][app][name]\" }\n rename => { \"[cf_space_id]\" => \"[@source][space][id]\" | |
| }\n rename => { \"[cf_space_name]\" => \"[@source][space][name]\" }\n | |
| \ rename => { \"[cf_org_id]\" => \"[@source][org][id]\" }\n rename | |
| => { \"[cf_org_name]\" => \"[@source][org][name]\" }\n\n rename => { | |
| \"[level]\" => \"[@level]\" }\n uppercase => [ \"[@level]\" ]\n\n rename | |
| => { \"[host]\" => \"[@source][host]\" }\n rename => { \"[source_type]\" | |
| => \"[@source][component]\" }\n rename => { \"[source_instance]\" => | |
| \"[@source][instance]\" }\n add_field => { \"[@source][name]\" => \"%{[@source][component]}/%{[@source][instance]}\" | |
| }\n convert => { \"[@source][instance]\" => \"integer\" }\n\n rename | |
| => { \"[message_type]\" => \"[@source][message_type]\" }\n rename => | |
| { \"[origin]\" => \"[@source][origin]\" }\n\n rename => { \"[msg]\" => | |
| \"[@message]\" }\n }\n\n # Replace the unicode newline character \\u2028 | |
| with \\n, which Kibana will display as a new line. Seems that passing a string | |
| with an actual newline in it is the only way to make gsub work\n mutate {\n | |
| \ gsub => [ \"[@message]\", '\\u2028', \"\n\"\n ]\n }\n\n if | |
| ('RTR' in [@source][component]) {\n grok {\n\n #cf-release | |
| > v222 - includes x_forwarded_proto\n match => { '@message' => '%{HOSTNAME:[RTR][hostname]} | |
| - \\[(?<time>%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\\] \\\"%{WORD:[RTR][verb]} | |
| %{URIPATHPARAM:[RTR][path]} %{PROG:[RTR][http_spec]}\\\" %{BASE10NUM:[RTR][status]:int} | |
| %{BASE10NUM:[RTR][request_bytes_received]:int} %{BASE10NUM:[RTR][body_bytes_sent]:int} | |
| \\\"%{GREEDYDATA:[RTR][referer]}\\\" \\\"%{GREEDYDATA:[RTR][http_user_agent]}\\\" | |
| %{HOSTPORT} x_forwarded_for:\\\"%{GREEDYDATA:[RTR][x_forwarded_for]}\\\" x_forwarded_proto:\\\"%{GREEDYDATA:[RTR][x_forwarded_proto]}\\\" | |
| vcap_request_id:%{NOTSPACE:[RTR][vcap_request_id]} response_time:%{NUMBER:[RTR][response_time_sec]:float} | |
| app_id:%{NOTSPACE}%{GREEDYDATA}' }\n\n #cf-release > v205 - includes | |
| RequestBytesReceived\n match => { '@message' => '%{HOSTNAME:[RTR][hostname]} | |
| - \\[(?<time>%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\\] \\\"%{WORD:[RTR][verb]} | |
| %{URIPATHPARAM:[RTR][path]} %{PROG:[RTR][http_spec]}\\\" %{BASE10NUM:[RTR][status]:int} | |
| %{BASE10NUM:[RTR][request_bytes_received]:int} %{BASE10NUM:[RTR][body_bytes_sent]:int} | |
| \\\"%{GREEDYDATA:[RTR][referer]}\\\" \\\"%{GREEDYDATA:[RTR][http_user_agent]}\\\" | |
| %{HOSTPORT} x_forwarded_for:\\\"%{GREEDYDATA:[RTR][x_forwarded_for]}\\\" vcap_request_id:%{NOTSPACE:[RTR][vcap_request_id]} | |
| response_time:%{NUMBER:[RTR][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}' | |
| }\n\n #cf-release <= v205\n match => { '@message' => '%{HOSTNAME:[RTR][hostname]} | |
| - \\[(?<time>%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\\] \\\"%{WORD:[RTR][verb]} | |
| %{URIPATHPARAM:[RTR][path]} %{PROG:[RTR][http_spec]}\\\" %{BASE10NUM:[RTR][status]:int} | |
| %{BASE10NUM:[RTR][body_bytes_sent]:int} \\\"%{GREEDYDATA:[RTR][referer]}\\\" | |
| \\\"%{GREEDYDATA:[RTR][http_user_agent]}\\\" %{HOSTPORT} x_forwarded_for:\\\"%{GREEDYDATA:[RTR][x_forwarded_for]}\\\" | |
| vcap_request_id:%{NOTSPACE:[RTR][vcap_request_id]} response_time:%{NUMBER:[RTR][response_time_sec]:float} | |
| app_id:%{NOTSPACE}%{GREEDYDATA}' }\n overwrite => [ \"time\" ]\n | |
| \ tag_on_failure => [ 'fail/firehose/RTR' ]\n add_tag => | |
| \"RTR\"\n }\n\n if !(\"fail/firehose/RTR\" in [tags]) {\n date | |
| {\n match => [ \"time\", \"dd/MM/y:HH:mm:ss Z\" ]\n }\n\n | |
| \ # set @level based on HTTP status\n if [RTR][status] | |
| >= 500 {\n mutate {\n replace => { \"[@level]\" | |
| => \"ERROR\" }\n }\n } else if [RTR][status] >= 400 | |
| {\n mutate {\n replace => { \"[@level]\" => | |
| \"WARN\" }\n }\n }\n\n\n if [RTR][x_forwarded_for] | |
| {\n mutate {\n gsub => [\"[RTR][x_forwarded_for]\",\"[\\s\\\\\"]\",\"\"] | |
| # remove quotes and whitespace\n split => [\"[RTR][x_forwarded_for]\", | |
| \",\"] # format is client, proxy1, proxy2 ...\n }\n\n ruby | |
| {\n code => \"event['RTR']['response_time_ms'] = (event['RTR']['response_time_sec']*1000).to_int\"\n | |
| \ remove_field => \"response_time_sec\"\n }\n\n | |
| \ mutate {\n add_field => [\"[RTR][remote_addr]\", | |
| \"%{[RTR][x_forwarded_for][0]}\"]\n }\n\n if ([RTR][remote_addr] | |
| =~ /([0-9]{1,3}\\.){3}[0-9]{1,3}/) {\n geoip {\n source | |
| => \"[RTR][remote_addr]\"\n }\n }\n }\n | |
| \ }\n }\n\n # Cleanup unused fields\n mutate {\n remove_field | |
| => \"time\"\n remove_field => \"timestamp\"\n remove_field => | |
| \"received_from\"\n remove_field => \"received_at\"\n remove_field | |
| => \"syslog_message\"\n remove_field => \"syslog_severity_code\"\n remove_field | |
| => \"syslog_facility_code\"\n remove_field => \"syslog_facility\"\n remove_field | |
| => \"syslog_severity\"\n remove_field => \"syslog_pri\"\n remove_field | |
| => \"syslog_program\"\n remove_field => \"syslog_pid\"\n remove_field | |
| => \"syslog_hostname\"\n remove_field => \"syslog_timestamp\"\n }\n\n | |
| \ #Route to index and type\n mutate {\n replace => { \"[@metadata][index]\" | |
| => \"app\" }\n replace => { \"[@metadata][type]\" => \"%{[event_type]}\" | |
| }\n remove_field => \"[event_type]\"\n add_tag => [ 'firehose' | |
| ]\n }\n\n}\n\n if \"firehose\" in [tags] {\n # Parse messages from /v2/events | |
| api logged by https://github.com/stayup-io/cf-app-events-logger \n\nif [@message] | |
| =~ /.*\"event_type\":\"AppEvent\".*/ {\n\n json {\n source => \"[@message]\"\n | |
| \ target => \"[app_event]\"\n add_tag => \"app_event\"\n }\n\n | |
| \ date {\n match => [ \"[app_event][timestamp]\", \"ISO8601\" ]\n }\n\n | |
| \ ruby {\n code => \"event['@metadata']['app_event_metadata_as_string'] | |
| = event['app_event']['metadata'].to_s\"\n }\n mutate {\n replace | |
| => { \"[@message]\" => \"%{[app_event][type]} %{[@metadata][app_event_metadata_as_string]}\" | |
| }\n }\n\n mutate {\n replace => { \"[@source][component]\" => \"AppEvent\" | |
| }\n replace => { \"[@source][instance]\" => \"0\" }\n convert | |
| => { \"[@source][instance]\" => \"integer\" }\n replace => { \"[@source][name]\" | |
| => \"AppEvent/0\" }\n\n replace => { \"[@source][app][id]\" => \"%{[app_event][app_id]}\" | |
| }\n replace => { \"[@source][app][name]\" => \"%{[app_event][app_name]}\" | |
| }\n replace => { \"[@source][space][id]\" => \"%{[app_event][space_guid]}\" | |
| }\n replace => { \"[@source][space][name]\" => \"%{[app_event][space_name]}\" | |
| }\n replace => { \"[@source][org][id]\" => \"%{[app_event][organization_guid]}\" | |
| }\n replace => { \"[@source][org][name]\" => \"%{[app_event][organization_name]}\" | |
| }\n\n replace => { \"[@metadata][type]\" => \"app_event\" }\n\n remove_field | |
| => [ \"[@source][message_type]\", \"[@source][origin]\", \"[@source][host]\" | |
| ]\n }\n\n}\n\n\n }\n} else if [@metadata][type] in [\"syslog\", \"relp\"] | |
| and [syslog_program] == \"vcap.uaa\" {\ngrok {\n match => { \"syslog_message\" | |
| =>\n \"\\[job=%{NOTSPACE:jobname}%{SPACE}index=%{NOTSPACE:jobindex}\\]%{SPACE}\\[%{TIMESTAMP_ISO8601:uaa_timestamp}\\]%{SPACE}uaa%{SPACE}-%{SPACE}%{NUMBER:pid:int}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}....%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}---%{SPACE}Audit:%{SPACE}%{WORD:audit_event_type}%{SPACE}\\('%{DATA:audit_event_data}'\\):%{SPACE}principal=%{DATA:audit_event_principal},%{SPACE}origin=\\[%{DATA:audit_event_origin}\\],%{SPACE}identityZoneId=\\[%{DATA:audit_event_identity_zone_id}\\]\"\n | |
| \ }\n tag_on_failure => [\n \"fail/cloudfoundry/uaa-audit\"\n ]\n | |
| \ add_tag => \"uaa-audit\"\n}\n\nif !(\"fail/cloudfoundry/uaa-audit\" in [tags]) | |
| {\n date {\n match => [ \"uaa_timestamp\", \"ISO8601\" ]\n remove_field | |
| => \"uaa_timestamp\"\n }\n\n if \"PrincipalAuthenticationFailure\" == | |
| [audit_event_type] {\n mutate {\n add_field => { \"audit_event_remote_address\" | |
| => \"%{audit_event_origin}\" }\n }\n }\n\n if [audit_event_origin] | |
| =~ /remoteAddress=/ {\n grok {\n match => { \"audit_event_origin\" | |
| => \"remoteAddress=%{IP:audit_event_remote_address}\" }\n }\n }\n\n | |
| \ if [audit_event_remote_address] {\n geoip {\n source => \"audit_event_remote_address\"\n | |
| \ }\n }\n\n mutate {\n remove_field => \"syslog_pri\"\n remove_field | |
| => \"syslog_facility\"\n remove_field => \"syslog_facility_code\"\n remove_field | |
| => \"syslog_message\"\n remove_field => \"syslog_severity\"\n remove_field | |
| => \"syslog_severity_code\"\n remove_field => \"syslog_program\"\n remove_field | |
| => \"syslog_timestamp\"\n remove_field => \"syslog_hostname\"\n\n split | |
| => { \"audit_event_origin\" => \", \" }\n \n add_field => { \"[@source][component]\" | |
| => \"UAA\" }\n add_field => { \"[@source][name]\" => \"%{jobname}/%{jobindex}\" | |
| }\n convert => { \"jobindex\" => \"integer\" }\n }\n \n mutate | |
| {\n remove_field => \"jobname\"\n rename => { \"jobindex\" => | |
| \"[@source][instance]\" }\n rename => { \"loglevel\" => \"[@level]\" | |
| }\n\n rename => { \"pid\" => \"[UAA][pid]\" }\n rename | |
| => { \"thread_name\" => \"[UAA][thread_name]\" }\n\n rename => { | |
| \"audit_event_type\" => \"[UAA][type]\" }\n rename => { | |
| \"audit_event_remote_address\" => \"[UAA][remote_address]\" }\n rename | |
| => { \"audit_event_data\" => \"[UAA][data]\" }\n rename | |
| => { \"audit_event_principal\" => \"[UAA][principal]\" }\n rename | |
| => { \"audit_event_origin\" => \"[UAA][origin]\" }\n rename | |
| => { \"audit_event_identity_zone_id\" => \"[UAA][identity_zone_id]\" }\n }\n\n | |
| \ #Route to index and type\n mutate {\n replace => { \"[@metadata][index]\" | |
| => \"platform\" }\n replace => { \"[@metadata][type]\" => \"uaa-audit\" | |
| }\n add_tag => \"uaa\"\n add_tag => \"audit\"\n }\n}\n\n\n} | |
| else if \"collector\" in [tags] {\n# Parse Cloud Foundry Collector\n\nmutate | |
| {\n add_field => { \"[@source][component]\" => \"%{[attributes][job]}\" }\n | |
| \ add_field => { \"[@source][instance]\" => \"%{[attributes][index]}\" }\n | |
| \ add_field => { \"[@source][deployment]\" => \"%{[attributes][deployment]}\" | |
| }\n add_field => { \"[@source][host]\" => \"%{[attributes][ip]}\" }\n remove_field | |
| => [ \"[attributes]\" ]\n}\n\nmutate {\n add_field => { \"[@source][name]\" | |
| => \"%{[@source][component]}/%{[@source][instance]}\" }\n}\n\nmutate {\n rename | |
| => { \"[key]\" => \"[metric][key]\" }\n convert => { \"[value]\" => \"string\" | |
| }\n}\n\nif [value] =~ /^[-+]?\\d+$/ {\n mutate { \n rename => { \"[value]\" | |
| => \"[metric][value_int]\" }\n convert => { \"[metric][value_int]\" => | |
| \"integer\" }\n }\n} else {\n mutate { \n rename => { \"[value]\" | |
| => \"[metric][value_float]\" }\n convert => { \"[metric][value_float]\" | |
| => \"float\" }\n }\n}\n\nmutate {\n remove_field => [ \"level\", \"facility\", | |
| \"file\", \"line\", \"version\", \"source_host\", \"host\" ]\n}\n\n#Route to | |
| index and type\nmutate {\n replace => { \"[@metadata][index]\" => \"platform\" | |
| }\n replace => { \"[@metadata][type]\" => \"metric\" }\n add_tag => \"metric\"\n}\n\n} | |
| else if [@metadata][type] in [\"syslog\", \"relp\"] and [syslog_program] =~ | |
| /vcap\\..*/ {\n# Parse Cloud Foundry logs from syslog_aggregator\n\ngrok {\n | |
| \ match => { \"syslog_message\" => \"(?:\\[job=%{NOTSPACE:[@job][name]}|-) | |
| +(?:index=%{NOTSPACE:[@job][index]}\\]|-) %{GREEDYDATA:_message_json}\" }\n | |
| \ tag_on_failure => [\n \"_grokparsefailure-cf-vcap\"\n ]\n}\n\nif | |
| !(\"_grokparsefailure-cf-vcap\" in [tags]) {\n kv {\n source => \"msgdata\"\n | |
| \ field_split => \" \"\n target => \"msgdata\"\n }\n\n json | |
| {\n source => \"_message_json\"\n remove_field => \"_message_json\"\n | |
| \ }\n\n mutate {\n rename => [ \"syslog_program\", \"[@shipper][name]\" | |
| ]\n replace => [ \"[@job][host]\", \"%{[@source][host]}\" ]\n gsub | |
| => [\n \"[@shipper][name]\", \"\\.\", \"_\",\n \"[@job][name]\", | |
| \"\\.\", \"_\"\n ]\n }\n\n if [source] == \"NatsStreamForwarder\" | |
| {\n json {\n source => \"[data][nats_message]\"\n target | |
| => \"nats_message\"\n }\n\n mutate {\n remove_field | |
| => \"[data][nats_message]\"\n }\n }\n\n mutate {\n add_tag | |
| => \"cloudfoundry_vcap\"\n replace => [ \"[@shipper][priority]\", \"%{syslog_pri}\" | |
| ]\n replace => [ \"[@shipper][name]\", \"%{[@shipper][name]}_%{[@metadata][type]}\" | |
| ]\n }\n\n mutate {\n remove_field => \"syslog_facility\"\n remove_field | |
| => \"syslog_facility_code\"\n remove_field => \"syslog_message\"\n remove_field | |
| => \"syslog_severity\"\n remove_field => \"syslog_severity_code\"\n }\n\n | |
| \ mutate {\n #replace => { \"[@metadata][index]\" => \"platform\" }\n | |
| \ replace => { \"[@metadata][type]\" => \"%{[@metadata][type]}_cf\" }\n | |
| \ }\n}\n\n}\n\n#Cleanup\nmutate {\n rename => { \"[tags]\" => \"[@tags]\" | |
| }\n remove_field => [ \"@version\" ]\n remove_field => \"@type\" \n}\n" | |
| releases: | |
| - name: logsearch-for-cloudfoundry | |
| version: latest |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment