Last active
April 10, 2017 07:03
-
-
Save calum-github/2332cf1757417c3569cdaf9527c91f4e to your computer and use it in GitHub Desktop.
Log stash config file for influxdb - pull data out of sqlite db, add some fields/tags and send to influx
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| input { | |
| jdbc { | |
| jdbc_driver_library => "/usr/local/logstash-5.2.1/plugins/sqlite-jdbc-3.16.1.jar" | |
| jdbc_driver_class => "org.sqlite.jdbc" | |
| jdbc_connection_string => "jdbc:sqlite:/Library/Server/Caching/Logs/Metrics.sqlite" | |
| jdbc_user => "" | |
| schedule => "* * * * *" | |
| statement => 'SELECT * FROM statsData WHERE metricname LIKE "bytes.%.toclients" OR metricname LIKE "bytes.%.topeers"' | |
| tracking_column => "entryindex" | |
| use_column_value => true | |
| type => "bytestoclients" | |
| } | |
| jdbc { | |
| jdbc_driver_library => "/usr/local/logstash-5.2.1/plugins/sqlite-jdbc-3.16.1.jar" | |
| jdbc_driver_class => "org.sqlite.jdbc" | |
| jdbc_connection_string => "jdbc:sqlite:/Library/Server/Caching/Logs/Metrics.sqlite" | |
| jdbc_user => "" | |
| schedule => "* * * * *" | |
| statement => 'SELECT * FROM statsData WHERE metricname LIKE "requests.%"' | |
| tracking_column => "entryindex" | |
| use_column_value => true | |
| type => "requests" | |
| } | |
| jdbc { | |
| jdbc_driver_library => "/usr/local/logstash-5.2.1/plugins/sqlite-jdbc-3.16.1.jar" | |
| jdbc_driver_class => "org.sqlite.jdbc" | |
| jdbc_connection_string => "jdbc:sqlite:/Library/Server/Caching/Logs/Metrics.sqlite" | |
| jdbc_user => "" | |
| schedule => "* * * * *" | |
| statement => 'SELECT * FROM statsData WHERE metricname LIKE "replies.%"' | |
| tracking_column => "entryindex" | |
| use_column_value => true | |
| type => "replies" | |
| } | |
| jdbc { | |
| jdbc_driver_library => "/usr/local/logstash-5.2.1/plugins/sqlite-jdbc-3.16.1.jar" | |
| jdbc_driver_class => "org.sqlite.jdbc" | |
| jdbc_connection_string => "jdbc:sqlite:/Library/Server/Caching/Logs/Metrics.sqlite" | |
| jdbc_user => "" | |
| schedule => "* * * * *" | |
| statement => 'SELECT * FROM statsData WHERE metricname LIKE "bytes.purged.%"' | |
| tracking_column => "entryindex" | |
| use_column_value => true | |
| type => "bytespurged" | |
| } | |
| jdbc { | |
| jdbc_driver_library => "/usr/local/logstash-5.2.1/plugins/sqlite-jdbc-3.16.1.jar" | |
| jdbc_driver_class => "org.sqlite.jdbc" | |
| jdbc_connection_string => "jdbc:sqlite:/Library/Server/Caching/Logs/Metrics.sqlite" | |
| jdbc_user => "" | |
| schedule => "* * * * *" | |
| statement => 'SELECT * FROM statsData WHERE metricname LIKE "bytes.dropped"' | |
| tracking_column => "entryindex" | |
| use_column_value => true | |
| type => "bytesdropped" | |
| } | |
| jdbc { | |
| jdbc_driver_library => "/usr/local/logstash-5.2.1/plugins/sqlite-jdbc-3.16.1.jar" | |
| jdbc_driver_class => "org.sqlite.jdbc" | |
| jdbc_connection_string => "jdbc:sqlite:/Library/Server/Caching/Logs/Metrics.sqlite" | |
| jdbc_user => "" | |
| schedule => "* * * * *" | |
| statement => 'SELECT * FROM statsData WHERE metricname LIKE "bytes.imported.%"' | |
| tracking_column => "entryindex" | |
| use_column_value => true | |
| type => "bytesimported" | |
| } | |
| jdbc { | |
| jdbc_driver_library => "/usr/local/logstash-5.2.1/plugins/sqlite-jdbc-3.16.1.jar" | |
| jdbc_driver_class => "org.sqlite.jdbc" | |
| jdbc_connection_string => "jdbc:sqlite:/Library/Server/Caching/Logs/Metrics.sqlite" | |
| jdbc_user => "" | |
| schedule => "* * * * *" | |
| statement => 'SELECT * FROM statsData WHERE metricname LIKE "imports.%"' | |
| tracking_column => "entryindex" | |
| use_column_value => true | |
| type => "imports" | |
| } | |
| } | |
| filter { | |
| if [type] == "bytestoclients" { | |
| mutate { | |
| add_tag => [ "bytestoclients" ] | |
| } | |
| } | |
| if [type] == "bytespurged" { | |
| mutate { | |
| add_tag => [ "bytespurged" ] | |
| } | |
| } | |
| if [type] == "bytesimported" { | |
| mutate { | |
| add_tag => [ "bytesimported" ] | |
| } | |
| } | |
| if [type] == "imports" { | |
| mutate { | |
| add_tag => [ "imports" ] | |
| } | |
| } | |
| if [type] == "bytesdropped" { | |
| mutate { | |
| add_tag => [ "bytesdropped" ] | |
| } | |
| } | |
| if [type] == "replies" { | |
| mutate { | |
| add_tag => [ "replies" ] | |
| } | |
| } | |
| if [type] == "requests" { | |
| mutate { | |
| add_tag => [ "requests" ] | |
| } | |
| } | |
| date { | |
| match => [ "collectiondate", "UNIX" ] | |
| remove_field => [ "collectiondate", "expirationdate" ] | |
| } | |
| mutate { | |
| add_field => { | |
| "region" => "Region 1" | |
| "site_name" => "Site Name Alpha" | |
| "site_code" => "1234" | |
| "school_type" => "High School" | |
| } | |
| } | |
| } | |
| output { | |
| if "bytestoclients" in [tags] { | |
| influxdb { | |
| allow_time_override => true | |
| host => "my.influxdb.server.com" | |
| measurement => "bytestoclients" | |
| idle_flush_time => 1 | |
| flush_size => 100 | |
| send_as_tags => [ "region", "site_name", "site_code", "school_type" ] | |
| data_points => { | |
| "%{metricname}" => "%{datavalue}" | |
| "region" => "%{region}" | |
| "site_name" => "%{site_name}" | |
| "site_code" => "%{site_code}" | |
| "school_type" => "%{school_type}" | |
| } | |
| coerce_values => { | |
| "bytes.fromcache.toclients" => "integer" | |
| "bytes.fromorigin.toclients" => "integer" | |
| "bytes.frompeers.toclients" => "integer" | |
| "bytes.fromorigin.topeers" => "integer" | |
| "bytes.fromcache.topeers" => "integer" | |
| } | |
| db => "caching" | |
| retention_policy => "autogen" | |
| } | |
| } | |
| if "bytespurged" in [tags] { | |
| influxdb { | |
| allow_time_override => true | |
| host => "my.influxdb.server.com" | |
| measurement => "bytespurged" | |
| idle_flush_time => 1 | |
| flush_size => 100 | |
| send_as_tags => [ "region", "site_name", "site_code", "school_type" ] | |
| data_points => { | |
| "%{metricname}" => "%{datavalue}" | |
| "region" => "%{region}" | |
| "site_name" => "%{site_name}" | |
| "site_code" => "%{site_code}" | |
| "school_type" => "%{school_type}" | |
| } | |
| coerce_values => { | |
| "bytes.purged.youngerthan1day" => "integer" | |
| "bytes.purged.youngerthan7days" => "integer" | |
| "bytes.purged.youngerthan30days" => "integer" | |
| "bytes.purged.total" => "integer" | |
| } | |
| db => "caching" | |
| retention_policy => "autogen" | |
| } | |
| } | |
| if "bytesimported" in [tags] { | |
| influxdb { | |
| allow_time_override => true | |
| host => "my.influxdb.server.com" | |
| measurement => "bytesimported" | |
| idle_flush_time => 1 | |
| flush_size => 100 | |
| send_as_tags => [ "region", "site_name", "site_code", "school_type" ] | |
| data_points => { | |
| "%{metricname}" => "%{datavalue}" | |
| "region" => "%{region}" | |
| "site_name" => "%{site_name}" | |
| "site_code" => "%{site_code}" | |
| "school_type" => "%{school_type}" | |
| } | |
| coerce_values => { | |
| "bytes.imported.byhttp" => "integer" | |
| "bytes.imported.byxpc" => "integer" | |
| } | |
| db => "caching" | |
| retention_policy => "autogen" | |
| } | |
| } | |
| if "imports" in [tags] { | |
| influxdb { | |
| allow_time_override => true | |
| host => "my.influxdb.server.com" | |
| measurement => "imports" | |
| idle_flush_time => 1 | |
| flush_size => 100 | |
| send_as_tags => [ "region", "site_name", "site_code", "school_type" ] | |
| data_points => { | |
| "%{metricname}" => "%{datavalue}" | |
| "region" => "%{region}" | |
| "site_name" => "%{site_name}" | |
| "site_code" => "%{site_code}" | |
| "school_type" => "%{school_type}" | |
| } | |
| coerce_values => { | |
| "imports.byhttp" => "integer" | |
| "imports.byxpc" => "integer" | |
| } | |
| db => "caching" | |
| retention_policy => "autogen" | |
| } | |
| } | |
| if "bytesdropped" in [tags] { | |
| influxdb { | |
| allow_time_override => true | |
| host => "my.influxdb.server.com" | |
| measurement => "bytesdropped" | |
| idle_flush_time => 1 | |
| flush_size => 100 | |
| send_as_tags => [ "region", "site_name", "site_code", "school_type" ] | |
| data_points => { | |
| "%{metricname}" => "%{datavalue}" | |
| "region" => "%{region}" | |
| "site_name" => "%{site_name}" | |
| "site_code" => "%{site_code}" | |
| "school_type" => "%{school_type}" | |
| } | |
| coerce_values => { | |
| "bytes.dropped" => "integer" | |
| } | |
| db => "caching" | |
| retention_policy => "autogen" | |
| } | |
| } | |
| if "replies" in [tags] { | |
| influxdb { | |
| allow_time_override => true | |
| host => "my.influxdb.server.com" | |
| measurement => "replies" | |
| idle_flush_time => 1 | |
| flush_size => 100 | |
| send_as_tags => [ "region", "site_name", "site_code", "school_type" ] | |
| data_points => { | |
| "%{metricname}" => "%{datavalue}" | |
| "region" => "%{region}" | |
| "site_name" => "%{site_name}" | |
| "site_code" => "%{site_code}" | |
| "school_type" => "%{school_type}" | |
| } | |
| coerce_values => { | |
| "replies.fromcache.toclients" => "integer" | |
| "replies.fromcache.topeers" => "integer" | |
| "replies.fromorigin.toclients" => "integer" | |
| "replies.fromorigin.topeers" => "integer" | |
| "replies.frompeers.toclients" => "integer" | |
| } | |
| db => "caching" | |
| retention_policy => "autogen" | |
| } | |
| } | |
| if "requests" in [tags] { | |
| influxdb { | |
| allow_time_override => true | |
| host => "my.influxdb.server.com" | |
| measurement => "requests" | |
| idle_flush_time => 1 | |
| flush_size => 100 | |
| send_as_tags => [ "region", "site_name", "site_code", "school_type" ] | |
| data_points => { | |
| "%{metricname}" => "%{datavalue}" | |
| "region" => "%{region}" | |
| "site_name" => "%{site_name}" | |
| "site_code" => "%{site_code}" | |
| "school_type" => "%{school_type}" | |
| } | |
| coerce_values => { | |
| "requests.frompeers" => "integer" | |
| "requests.fromclients" => "integer" | |
| } | |
| db => "caching" | |
| retention_policy => "autogen" | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment