Last active
March 19, 2020 12:48
-
-
Save iamhowardtheduck/b27d433eb5629b47c07bffd20c64eef7 to your computer and use it in GitHub Desktop.
Coronavirus "COVID-19-john.conf" Logstash Pipeline
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| input { | |
| # Use file input to collect all CSVs from "git clone https://github.com/CSSEGISandData/COVID-19.git" | |
| # Simply run the command while in /var/log: git clone https://github.com/CSSEGISandData/COVID-19.git | |
| file { | |
| path => "/var/log/COVID-19/csse_covid_19_data/csse_covid_19_daily_reports/*.csv" | |
| start_position => "beginning" | |
| } } | |
| filter { | |
| csv { | |
| # Use the CSV filter to capture data | |
| skip_empty_columns => true | |
| skip_empty_rows => true | |
| # Skip omitted info | |
| autodetect_column_names => true | |
| # Just in case it mixes up on us | |
| columns => ["Province-State","Country","LastDate","Confirmed","Deaths","Recovered","Latitude","Longitude"] | |
| convert => { | |
| "Confirmed" => "integer" | |
| "Deaths" => "integer" | |
| "Recovered" => "integer" | |
| "Latitude" => "float" | |
| "Longitude" => "float" | |
| # Set the field values now, we will get errors when no data is present while setting the geo_point but that's ok | |
| } | |
| } | |
| mutate { add_field => { "corona.geo_point" => "%{Latitude},%{Longitude}" } } | |
| # Create the geo_point field for mapping plots | |
| date { match => [ "LastDate", "ISO8601" ] } | |
| # Set the date so we can use it for search | |
| grok { match => [ | |
| "Province-State", "\s?(?<Region>[^\,]+)\,\s(?<State>[^\(]+)", | |
| "Province-State", "\s?(?<Region>[^\,]+)\,\s(?<State>[^\(]+)\s\((?<Origin>[^\)]+)\)", | |
| "Province-State", "(?<State>Unassigned Location)\s\((?<Origin>[^\)]+)\)", | |
| "Province-State", "(?<State>Diamond Princess cruise ship)\s\((?<Origin>[^\)]+)\)", | |
| "Province-State", "\s?(?<Country>occupied Palestinian territory)", | |
| "Province-State", "\s?(?<Region>[^\,]+)" | |
| ]} | |
| # Parse out the fied Province-State into more meaningful data, parser errors will occur when there is no Province-State data present | |
| # The if's below set the Country field to the 2-character expression | |
| # Updated 2020-03-19 | |
| if [Deaths] == "Deaths" { drop { }} | |
| if [Country] == "Zambia" { mutate { replace => { "Country" => "ZM" } } } | |
| if [Country] == "Mauritius" { mutate { replace => { "Country" => "MU" } } } | |
| if [Country] == "Kyrgyzstan" { mutate { replace => { "Country" => "KG" } } } | |
| if [Country] == "Djibouti" { mutate { replace => { "Country" => "DJ" } } } | |
| if [Country] == "Gambia,The" { mutate { replace => { "Country" => "GM" } } } | |
| if [Country] == "The Gambia" { mutate { replace => { "Country" => "GM" } } } | |
| if [Country] == "Montenegro" { mutate { replace => { "Country" => "ME" } } } | |
| if [Country] == "Barbados" { mutate { replace => { "Country" => "BB" } } } | |
| if [Country] == "The Bahamas" { mutate { replace => { "Country" => "BS" } } } | |
| if [Country] == "Tanzania" { mutate { replace => { "Country" => "TZ" } } } | |
| if [Country] == "Somalia" { mutate { replace => { "Country" => "SO" } } } | |
| if [Country] == "Republic of the Congo" { mutate { replace => { "Country" => "CD" } } } | |
| if [Country] == "Puerto Rico" { mutate { replace => { "Country" => "PR" } } } | |
| if [Country] == "Mayotte" { mutate { replace => { "Country" => "YT" } } } | |
| if [Country] == "Liberia" { mutate { replace => { "Country" => "LR" } } } | |
| if [Country] == "Greenland" { mutate { replace => { "Country" => "GL" } } } | |
| if [Country] == "Guam" { mutate { replace => { "Country" => "GU" } } } | |
| if [Country] == "Benin" { mutate { replace => { "Country" => "BJ" } } } | |
| if [Country] == "Equatorial Guinea" { mutate { replace => { "Country" => "GQ" } } } | |
| if [Country] == "Uzbekistan" { mutate { replace => { "Country" => "UZ" } } } | |
| if [Country] == "Kosovo" { mutate { replace => { "Country" => "XK" } } } | |
| if [Country] == "Congo (Brazzaville)" { mutate { replace => { "Country" => "CG" } } } | |
| if [Country] == "Central African Republic" { mutate { replace => { "Country" => "CF" } } } | |
| if [Country] == "Venezuela" { mutate { replace => { "Country" => "VE" } } } | |
| if [Country] == "Uruguay" { mutate { replace => { "Country" => "UY" } } } | |
| if [Country] == "Trinidad and Tobago" { mutate { replace => { "Country" => "TT" } } } | |
| if [Country] == "Suriname" { mutate { replace => { "Country" => "SR" } } } | |
| if [Country] == "Seychelles" { mutate { replace => { "Country" => "SC" } } } | |
| if [Country] == "Saint Vincent and the Grenadines" { mutate { replace => { "Country" => "VC" } } } | |
| if [Country] == "Saint Lucia" { mutate { replace => { "Country" => "LC" } } } | |
| if [Country] == "Rwanda" { mutate { replace => { "Country" => "RW" } } } | |
| if [Country] == "Namibia" { mutate { replace => { "Country" => "NA" } } } | |
| if [Country] == "Mauritania" { mutate { replace => { "Country" => "MR" } } } | |
| if [Country] == "Jersey" { mutate { replace => { "Country" => "JE" } } } | |
| if [Country] == "Guernsey" { mutate { replace => { "Country" => "GG" } } } | |
| if [Country] == "Guatemala" { mutate { replace => { "Country" => "GT" } } } | |
| if [Country] == "Ghana" { mutate { replace => { "Country" => "GH" } } } | |
| if [Country] == "Gabon" { mutate { replace => { "Country" => "GS" } } } | |
| if [Country] == "Eswatini" { mutate { replace => { "Country" => "SZ" } } } | |
| if [Country] == "Curacao" { mutate { replace => { "Country" => "CW" } } } | |
| if [Country] == "Sudan" { mutate { replace => { "Country" => "SD" } } } | |
| if [Country] == "Kenya" { mutate { replace => { "Country" => "KE" } } } | |
| if [Country] == "Kazakhstan" { mutate { replace => { "Country" => "KZ" } } } | |
| if [Country] == "Guinea" { mutate { replace => { "Country" => "GN" } } } | |
| if [Country] == "Guadeloupe" { mutate { replace => { "Country" => "GP" } } } | |
| if [Country] == "Ethiopia" { mutate { replace => { "Country" => "ET" } } } | |
| if [Country] == "Cayman Islands" { mutate { replace => { "Country" => "KY" } } } | |
| if [Country] == "Aruba" { mutate { replace => { "Country" => "AW" } } } | |
| if [Country] == "Antigua and Barbuda" { mutate { replace => { "Country" => "AG" } } } | |
| if [Country] == "Guyana" { mutate { replace => { "Country" => "GY" } } } | |
| if [Country] == "Cuba" { mutate { replace => { "Country" => "CU" } } } | |
| if [Country] == "Taiwan*" { mutate { replace => { "Country" => "TW" } } } | |
| if [Country] == "Taipei and environs" { mutate { replace => { "Country" => "TW" } } } | |
| if [Region] == "Taiwan" { mutate { replace => { "Country" => "TW" } } } | |
| if [Country] == "Bolivia" { mutate { replace => { "Country" => "BO" } } } | |
| if [Country] == "Congo (Kinshasa)" { mutate { replace => { "Country" => "CD" } } } | |
| if [Country] == "Cote d'Ivoire" { mutate { replace => { "Country" => "CI" } } } | |
| if [Country] == "Honduras" { mutate { replace => { "Country" => "HN" } } } | |
| if [Country] == "Jamaica" { mutate { replace => { "Country" => "JM" } } } | |
| if [Country] == "Republic of Ireland" { mutate { replace => { "Country" => "IE" } } } | |
| if [Country] == "Reunion" { mutate { replace => { "Country" => "RE" } } } | |
| if [Country] == "Turkey" { mutate { replace => { "Country" => "TR" } } } | |
| if [Country] == "Viet Nam" { mutate { replace => { "Country" => "VN" } } } | |
| if [Country] == "United Kingdom" { mutate { replace => { "Country" => "GB" } } } | |
| if [Country] == "Taipei*" { mutate { replace => { "Region" => "Taiwan" } } } | |
| if [Country] == "Taipei and environs" { mutate { replace => { "Region" => "Taiwan" } } } | |
| if [Country] == "St. Martin" { mutate { replace => { "Country" => "MF" } } } | |
| if [Country] == "St. Martin" { mutate { replace => { "Country" => "MF" } } } | |
| if [Country] == "Saint Martin" { mutate { replace => { "Country" => "MF" } } } | |
| if [Country] == "Russian Federation" { mutate { replace => { "Country" => "RU" } } } | |
| if [Country] == "Republic of Moldova" { mutate { replace => { "Country" => "MD" } } } | |
| if [Country] == "Paraguay" { mutate { replace => { "Country" => "PY" } } } | |
| if [Country] == "Panama" { mutate { replace => { "Country" => "PA" } } } | |
| if [Country] == "Palestine" { mutate { replace => { "Country" => "PS" } } } | |
| if [Country] == "Mongolia" { mutate { replace => { "Country" => "MN" } } } | |
| if [Country] == "Moldova" { mutate { replace => { "Country" => "MD" } } } | |
| if [Country] == "Maldives" { mutate { replace => { "Country" => "MV" } } } | |
| if [Country] == "Hong Kong SAR" { mutate { replace => { "Country" => "HK" } } } | |
| if [Country] == "Macao SAR" { mutate { replace => { "Country" => "MO" } } } | |
| if [Country] == "Iran (Islamic Republic of)" { mutate { replace => { "Country" => "IR" } } } | |
| if [Country] == "Holy See" { mutate { replace => { "Country" => "VA" } } } | |
| if [Country] == "Czechia" { mutate { replace => { "Country" => "CZ" } } } | |
| if [Country] == "Cyprus" { mutate { replace => { "Country" => "CY" } } } | |
| if [Country] == "Cruise Ship" { mutate { replace => { "Origin" => "Cruise Ship" } } } | |
| if [Country] == "Channel Islands" { mutate { replace => { "Country" => "JE" } } } | |
| if [Country] == "Burkina Faso" { mutate { replace => { "Country" => "BF" } } } | |
| if [Country] == "Bulgaria" { mutate { replace => { "Country" => "BG" } } } | |
| if [Country] == "Brunei" { mutate { replace => { "Country" => "BN" } } } | |
| if [Country] == "Bangladesh" { mutate { replace => { "Country" => "BD" } } } | |
| if [Country] == "Albania" { mutate { replace => { "Country" => "AL" } } } | |
| if [Country] == "Mainland China" { mutate { replace => { "Country" => "CN" } } } | |
| if [Country] == "China" { mutate { replace => { "Country" => "CN" } } } | |
| if [Country] == "Korea, South" { mutate { replace => { "Country" => "KR" } } } | |
| if [Country] == "Republic of Korea" { mutate { replace => { "Country" => "KR" } } } | |
| if [Country] == "South Korea" { mutate { replace => { "Country" => "KR" } } } | |
| if [Country] == "Iran" { mutate { replace => { "Country" => "IR" } } } | |
| if [Country] == "Italy" { mutate { replace => { "Country" => "IT" } } } | |
| if [Country] == "Others" { mutate { replace => { "Country" => "Others" } } } | |
| if [Country] == "Germany" { mutate { replace => { "Country" => "DE" } } } | |
| if [Country] == "France" { mutate { replace => { "Country" => "FR" } } } | |
| if [Country] == "Japan" { mutate { replace => { "Country" => "JP" } } } | |
| if [Country] == "Spain" { mutate { replace => { "Country" => "ES" } } } | |
| if [Country] == "Switzerland" { mutate { replace => { "Country" => "CH" } } } | |
| if [Country] == "UK" { mutate { replace => { "Country" => "GB" } } } | |
| if [Country] == "Singapore" { mutate { replace => { "Country" => "SG" } } } | |
| if [Country] == "Netherlands" { mutate { replace => { "Country" => "NL" } } } | |
| if [Country] == "Belgium" { mutate { replace => { "Country" => "BE" } } } | |
| if [Country] == "Norway" { mutate { replace => { "Country" => "NO" } } } | |
| if [Country] == "Hong Kong" { mutate { replace => { "Country" => "HK" } } } | |
| if [Country] == "Sweden" { mutate { replace => { "Country" => "SE" } } } | |
| if [Country] == "Malaysia" { mutate { replace => { "Country" => "MY" } } } | |
| if [Country] == "Bahrain" { mutate { replace => { "Country" => "BS" } } } | |
| if [Country] == "Kuwait" { mutate { replace => { "Country" => "KW" } } } | |
| if [Country] == "US" { mutate { replace => { "Country" => "US" } } } | |
| if [Country] == "Austria" { mutate { replace => { "Country" => "AT" } } } | |
| if [Country] == "Thailand" { mutate { replace => { "Country" => "TH" } } } | |
| if [Country] == "Greece" { mutate { replace => { "Country" => "GR" } } } | |
| if [Country] == "Taiwan" { mutate { replace => { "Country" => "TW" } } } | |
| if [Country] == "Iceland" { mutate { replace => { "Country" => "IS" } } } | |
| if [Country] == "Iraq" { mutate { replace => { "Country" => "IQ" } } } | |
| if [Country] == "India" { mutate { replace => { "Country" => "IN" } } } | |
| if [Country] == "United Arab Emirates" { mutate { replace => { "Country" => "AE" } } } | |
| if [Country] == "Australia" { mutate { replace => { "Country" => "AU" } } } | |
| if [Country] == "Canada" { mutate { replace => { "Country" => "CA" } } } | |
| if [Country] == "Denmark" { mutate { replace => { "Country" => "DK" } } } | |
| if [Country] == "Lebanon" { mutate { replace => { "Country" => "LB" } } } | |
| if [Country] == "Israel" { mutate { replace => { "Country" => "IL" } } } | |
| if [Country] == "San Marino" { mutate { replace => { "Country" => "SM" } } } | |
| if [Country] == "Czech Republic" { mutate { replace => { "Country" => "CN" } } } | |
| if [Country] == "Ireland" { mutate { replace => { "Country" => "CZ" } } } | |
| if [Country] == "Mainland China" { mutate { replace => { "Country" => "IE" } } } | |
| if [Country] == "Algeria" { mutate { replace => { "Country" => "DZ" } } } | |
| if [Country] == "Oman" { mutate { replace => { "Country" => "OM" } } } | |
| if [Country] == "occupied Palestinian territory" { mutate { replace => { "Country" => "PS" } } } | |
| if [Country] == "Vietnam" { mutate { replace => { "Country" => "VN" } } } | |
| if [Country] == "Egypt" { mutate { replace => { "Country" => "EG" } } } | |
| if [Country] == "Finland" { mutate { replace => { "Country" => "FI" } } } | |
| if [Country] == "Brazil" { mutate { replace => { "Country" => "BR" } } } | |
| if [Country] == "Ecuador" { mutate { replace => { "Country" => "EC" } } } | |
| if [Country] == "Portugal" { mutate { replace => { "Country" => "PT" } } } | |
| if [Country] == "Russia" { mutate { replace => { "Country" => "RU" } } } | |
| if [Country] == "Croatia" { mutate { replace => { "Country" => "HR" } } } | |
| if [Country] == "Estonia" { mutate { replace => { "Country" => "EE" } } } | |
| if [Country] == "Macau" { mutate { replace => { "Country" => "MO" } } } | |
| if [Country] == "Romania" { mutate { replace => { "Country" => "RO" } } } | |
| if [Country] == "Qatar" { mutate { replace => { "Country" => "QA" } } } | |
| if [Country] == "Slovenia" { mutate { replace => { "Country" => "SI" } } } | |
| if [Country] == "Azerbaijan" { mutate { replace => { "Country" => "AZ" } } } | |
| if [Country] == "Belarus" { mutate { replace => { "Country" => "BY" } } } | |
| if [Country] == "Mexico" { mutate { replace => { "Country" => "MX" } } } | |
| if [Country] == "Pakistan" { mutate { replace => { "Country" => "PK" } } } | |
| if [Country] == "Philippines" { mutate { replace => { "Country" => "PH" } } } | |
| if [Country] == "Poland" { mutate { replace => { "Country" => "PL" } } } | |
| if [Country] == "Saudi Arabia" { mutate { replace => { "Country" => "SA" } } } | |
| if [Country] == "Chile" { mutate { replace => { "Country" => "CL" } } } | |
| if [Country] == "Georgia" { mutate { replace => { "Country" => "GE" } } } | |
| if [Country] == "Indonesia" { mutate { replace => { "Country" => "ID" } } } | |
| if [Country] == "New Zealand" { mutate { replace => { "Country" => "NZ" } } } | |
| if [Country] == "Senegal" { mutate { replace => { "Country" => "SN" } } } | |
| if [Country] == "North Macedonia" { mutate { replace => { "Country" => "MK" } } } | |
| if [Country] == "Saint Barthelemy" { mutate { replace => { "Country" => "BL" } } } | |
| if [Country] == "Argentina" { mutate { replace => { "Country" => "AR" } } } | |
| if [Country] == "Bosnia and Herzegovina" { mutate { replace => { "Country" => "BA" } } } | |
| if [Country] == "Dominican Republic" { mutate { replace => { "Country" => "DO" } } } | |
| if [Country] == "Hungary" { mutate { replace => { "Country" => "HU" } } } | |
| if [Country] == "Luxembourg" { mutate { replace => { "Country" => "LU" } } } | |
| if [Country] == "Morocco" { mutate { replace => { "Country" => "MA" } } } | |
| if [Country] == "Afghanistan" { mutate { replace => { "Country" => "AF" } } } | |
| if [Country] == "Andorra" { mutate { replace => { "Country" => "AD" } } } | |
| if [Country] == "Armenia" { mutate { replace => { "Country" => "AM" } } } | |
| if [Country] == "Bhutan" { mutate { replace => { "Country" => "BT" } } } | |
| if [Country] == "Cambodia" { mutate { replace => { "Country" => "KH" } } } | |
| if [Country] == "Cameroon" { mutate { replace => { "Country" => "CM" } } } | |
| if [Country] == "Colombia" { mutate { replace => { "Country" => "CN" } } } | |
| if [Country] == "Costa Rica" { mutate { replace => { "Country" => "CO" } } } | |
| if [Country] == "Faroe Islands" { mutate { replace => { "Country" => "CR" } } } | |
| if [Country] == "Gibraltar" { mutate { replace => { "Country" => "FO" } } } | |
| if [Country] == "Jordan" { mutate { replace => { "Country" => "GI" } } } | |
| if [Country] == "Latvia" { mutate { replace => { "Country" => "JO" } } } | |
| if [Country] == "Liechtenstein" { mutate { replace => { "Country" => "LV" } } } | |
| if [Country] == "Lithuania" { mutate { replace => { "Country" => "LI" } } } | |
| if [Country] == "Mainland China" { mutate { replace => { "Country" => "LT" } } } | |
| if [Country] == "Monaco" { mutate { replace => { "Country" => "MC" } } } | |
| if [Country] == "Nepal" { mutate { replace => { "Country" => "NP" } } } | |
| if [Country] == "Nigeria" { mutate { replace => { "Country" => "NG" } } } | |
| if [Country] == "Peru" { mutate { replace => { "Country" => "PE" } } } | |
| if [Country] == "Serbia" { mutate { replace => { "Country" => "RS" } } } | |
| if [Country] == "Slovakia" { mutate { replace => { "Country" => "SK" } } } | |
| if [Country] == "South Africa" { mutate { replace => { "Country" => "ZA" } } } | |
| if [Country] == "Sri Lanka" { mutate { replace => { "Country" => "LK" } } } | |
| if [Country] == "Togo" { mutate { replace => { "Country" => "TG" } } } | |
| if [Country] == "Tunisia" { mutate { replace => { "Country" => "TN" } } } | |
| if [Country] == "United States" { mutate { replace => { "Country" => "US" } } } | |
| if [Country] == "Ukraine" { mutate { replace => { "Country" => "UA" } } } | |
| if [Country] == "Vatican City" { mutate { replace => { "Country" => "VA" } } } | |
| if [Country] == "Malta" { mutate { replace => { "Country" => "MT" } } } | |
| if [Country] == "Martinique" { mutate { replace => { "Country" => "MQ" } } } | |
| if [Country] == "French Guiana" { mutate { replace => { "Country" => "GF" } } } | |
| if [Region] == "Alberta" { mutate { replace => { "State" => "AB" } } } | |
| if [Region] == "British Columbia" { mutate { replace => { "State" => "BC" } } } | |
| if [Region] == "Manitoba" { mutate { replace => { "State" => "MB" } } } | |
| if [Region] == "New Brunswick" { mutate { replace => { "State" => "NB" } } } | |
| if [Region] == "Newfoundland and Labrador" { mutate { replace => { "State" => "NL" } } } | |
| if [Region] == "Nova Scotia" { mutate { replace => { "State" => "NS" } } } | |
| if [Region] == "Northwest Territories" { mutate { replace => { "State" => "NT" } } } | |
| if [Region] == "Nunavut" { mutate { replace => { "State" => "NU" } } } | |
| if [Region] == "Ontario" { mutate { replace => { "State" => "ON" } } } | |
| if [Region] == "Prince Edward Island" { mutate { replace => { "State" => "PE" } } } | |
| if [Region] == "Quebec" { mutate { replace => { "State" => "QC" } } } | |
| if [Region] == "Saskatchewan" { mutate { replace => { "State" => "SK" } } } | |
| if [Region] == "Yukon" { mutate { replace => { "State" => "YT" } } } | |
| if [Region] == "Virgin Islands" { mutate { replace => { "State" => "VI" } } } | |
| if [Region] == "Washington" { mutate { replace => { "State" => "DC" } } } | |
| } | |
| # Replace *'s with your info | |
| output { | |
| elasticsearch { | |
| hosts => ["***********"] | |
| index => "covid-19-john" | |
| ilm_enabled => true | |
| ilm_policy => "Coronavirus" | |
| manage_template => false | |
| user => "********" | |
| password => "*******" | |
| ssl_certificate_verification => false | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment