Skip to content

Instantly share code, notes, and snippets.

@ibspoof

ibspoof/11-cassandra-filter.conf

Last active January 6, 2021 12:23
Show Gist options
  • Save ibspoof/917a888adb08a819eab7163b97e018cb to your computer and use it in GitHub Desktop.
Save ibspoof/917a888adb08a819eab7163b97e018cb to your computer and use it in GitHub Desktop.
Download ZIP
Grok Patterns and configuration for Cassandra logs
Raw
11-cassandra-filter.conf
filter {
if [type] == "cassandralog" {
grok {
break_on_match => true
match => {
"message" => [
"%{CASS_COMPACTION_LARGE_KEY}",
"%{CASS_SLAB_POOL_CLEANER_1}",
"%{CASS_SLAB_POOL_CLEANER_2}",
"%{CASS_MEMTABLE_FLUSH_START}",
"%{CASS_MEMTABLE_FLUSH_COMPLETE}",
"%{CASS_COMPACTION_COMPLETE}",
"%{CASS_GC_GRACE}",
"%{CASS_SERVICE_THREAD_PENDING}"
]
}
add_tag => [ "cass_parsed" ]
}
if "cass_parsed" not in [tags] {
grok {
match => { "message" => ["%{CASS_DEFAULT}"] }
add_tag => [ "cass_default" ]
}
}
if [fields][timezone] == "pst" {
## date is PST just parse it correctly
date {
locale => "en"
match => ["timestamp", "YYYY-MM-dd HH:mm:ss,SSS"] # 2016-08-16 04:13:05,063
timezone => "America/Los_Angeles"
target => "@timestamp"
remove_field => "timestamp"
# add_field => { "debug" => "tsMatched"}
}
} else if [fields][timezone] == "est" {
## date is EST
date {
locale => "en"
match => ["timestamp", "YYYY-MM-dd HH:mm:ss,SSS"] # 2016-08-16 04:13:05,063
timezone => "America/New_York"
target => "@timestamp"
remove_field => "timestamp"
# add_field => { "debug" => "tsMatched"}
}
} else {
## date is UTC just parse it correctly
date {
locale => "en"
match => ["timestamp", "YYYY-MM-dd HH:mm:ss,SSS"] # 2016-08-16 04:13:05,063
timezone => "Etc/UTC"
target => "@timestamp"
remove_field => "timestamp"
# add_field => { "debug" => "tsMatched"}
}
}
mutate {
convert => {
"line_number" => "integer"
"threadId" => "integer"
"bytes_in" => "integer"
"bytes_out" => "integer"
"time_ms" => "integer"
"bytes_onheap" => "integer"
"bytes_offheap" => "integer"
"commitlog_pos" => "integer"
"eden_orig_bytes" => "integer"
"eden_new_bytes" => "integer"
"oldgen_orig_bytes" => "integer"
"oldgen_new_bytes" => "integer"
"survivor_new_bytes" => "integer"
"survivor_orig_bytes" => "integer"
"commitlog_segid" => "integer"
"ops" => "integer"
"pkeys_in" => "integer"
"pkeys_out" => "integer"
"percent_of_orig" => "integer"
"cache_size" => "integer"
"cache_used" => "integer"
"threads_active" => "integer"
"threads_pending" => "integer"
"threads_blocked" => "integer"
"threads_completed" => "integer"
"threads_all_time_blocked" => "integer"
"size_kb" => "float"
"size_mb" => "float"
"total_onheap" => "float"
"total_offheap" => "float"
"live_onheap" => "float"
"live_offheap" => "float"
"flushing_onheap" => "float"
"flushing_offheap" => "float"
"this_onheap" => "float"
"this_offheap" => "float"
"percent_onheap" => "float"
"percent_offheap" => "float"
"speed_mb" => "float"
}
}
if [msg] {
mutate {
replace => [ "message", "%{msg}" ]
remove_field => "msg"
}
}
}
}
Raw
cassandra_grok_patterns
MILLISECOND (\d{3})
JAVALOGBACKTIMESTAMP %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:%{MINUTE}:%{SECOND},%{MILLISECOND}
CASS_BASE ^%{LOGLEVEL:level} \[(%{DATA:process}:%{INT:threadId}|%{DATA:process})\] %{JAVALOGBACKTIMESTAMP:timestamp} %{WORD:java_file}.java:%{INT:line_number} -
FLUSHSIZE %{BASE10NUM}(KiB|GiB|MiB)
# WARN [MemtableFlushWriter:20342] 2016-06-29 08:35:35,936 SSTableWriter.java:240 - Compacting large partition limo/read_audit_history:2016-6-29 (115624250 bytes)
CASS_COMPACTION_LARGE_KEY %{CASS_BASE} %{DATA:msg} %{WORD:keyspace}\/%{WORD:table}:%{DATA:partition_key} \(%{INT:size_bytes} bytes\)
# INFO [SlabPoolCleaner] 2016-06-28 16:13:14,966 ColumnFamilyStore.java:1211 - Flushing largest CFS(Keyspace='order_store', ColumnFamily='orderdata') to free up room. Used total: 0.11/0.00, live: 0.11/0.00, flushing: 0.00/0.00, this: 0.10/0.10
CASS_SLAB_POOL_CLEANER_1 %{CASS_BASE} %{DATA:msg}\(Keyspace='%{WORD:keyspace}', ColumnFamily='%{WORD:table}'\) %{GREEDYDATA}: %{BASE10NUM:total_onheap}/%{BASE10NUM:total_offheap}, live: %{BASE10NUM:live_onheap}/%{BASE10NUM:live_offheap}, flushing: %{BASE10NUM:flushing_onheap}/%{BASE10NUM:flushing_offheap}, this: %{BASE10NUM:this_onheap}/%{BASE10NUM:this_offheap}
# INFO [SlabPoolCleaner] 2016-06-28 16:13:14,967 ColumnFamilyStore.java:905 - Enqueuing flush of orderdata: 216588472 (10%) on-heap, 0 (0%) off-heap
CASS_SLAB_POOL_CLEANER_2 %{CASS_BASE} %{DATA:msg} of %{WORD:keyspace}: %{INT:bytes_onheap}%{DATA} \(%{INT:percent_onheap}%\) on-heap, %{INT:bytes_offheap} \(%{INT:percent_offheap}%\) %{GREEDYDATA}
# INFO [MemtableFlushWriter:320] 2016-06-28 16:13:14,969 Memtable.java:347 - Writing Memtable-orderdata@849528272(129.804MiB serialized bytes, 297813 ops, 10%/0% of on/off-heap limit)
CASS_MEMTABLE_FLUSH_START %{CASS_BASE} %{DATA:msg}-%{WORD:keyspace}%{DATA}\(%{BASE10NUM:size_mb}%{DATA}, %{INT:ops} ops, %{INT:percent_onheap}%/%{INT:percent_offheap}% %{GREEDYDATA}
# INFO [MemtableFlushWriter:1] 2016-06-28 16:19:48,627 Memtable.java:382 - Completed flushing /app/cassandra/datastax/dse-data01/system/local-7ad54392bcdd35a684174e047860b377/system-local-tmp-ka-3981-Data.db (0.000KiB) for commitlog position ReplayPosition(segmentId=1467130786324, position=567)
CASS_MEMTABLE_FLUSH_COMPLETE %{CASS_BASE} %{DATA:msg} %{UNIXPATH:sstable} \(%{FLUSHSIZE:size}\) %{DATA} %{GREEDYDATA:commitlog}
# INFO [CompactionExecutor:26] 2016-06-28 14:38:40,545 CompactionTask.java:274 - Compacted 2 sstables to [/app/cassandra/datastax/dse-data03/order_store/orderhistory-65765df1fc1f11e5974cd5203249f77a/order_store-orderhistory-ka-139517,]. 32,011,749 bytes to 29,390,144 (~91% of original) in 1,771ms = 15.826440MB/s. 39,261 total partitions merged to 36,402. Partition merge counts were {1:33543, 2:2859, }
CASS_COMPACTION_COMPLETE %{CASS_BASE} %{DATA:msg} to \[%{GREEDYDATA:sstable}\]. %{DATA:bytes_in} bytes to %{DATA:bytes_out} \(~%{INT:percent_of_orig}% of original\) in %{DATA:time_ms}ms = %{BASE10NUM:speed_mb}MB/s. %{DATA:pkeys_in} total partitions merged to %{DATA:pkeys_out}. %{GREEDYDATA:merge_cnt}
# WARN [Service Thread] 2016-06-30 14:42:41,538 GCInspector.java:256 - G1 Young Generation GC in 202ms. G1 Eden Space: 1157627904 -> 0; G1 Old Gen: 6819034128 -> 4151661080; G1 Survivor Space: 553648128 -> 218103808;
# WARN [Service Thread] 2016-06-30 14:42:41,538 GCInspector.java:256 - G1 Young Generation GC in 211ms. G1 Eden Space: 16894656512 -> 0; G1 Old Gen: 13259744800 -> 13267292200;
CASS_GC_GRACE %{CASS_BASE} %{DATA:msg} in %{INT:time_ms}ms. %{DATA}: %{INT:eden_orig_bytes} -> %{INT:eden_new_bytes}; %{DATA}: %{INT:oldgen_orig_bytes} -> %{INT:oldgen_new_bytes};( %{DATA}: %{INT:survivor_orig_bytes} -> %{INT:survivor_new_bytes};|)
# INFO [Service Thread] 2016-06-30 14:42:41,542 StatusLogger.java:99 - KeyCache 365405196 1073741824 all
# INFO [Service Thread] 2016-06-30 14:42:41,541 StatusLogger.java:66 - PendingRangeCalculator 0 0 79 0 0
# INFO [Service Thread] 2016-06-30 14:42:41,542 StatusLogger.java:75 - CompactionManager 2 5
CASS_SERVICE_THREAD_PENDING %{CASS_BASE} %{WORD:pool}%{SPACE}(%{INT:cache_used}%{SPACE}%{INT:cache_size}%{SPACE}all|%{INT:threads_active}%{SPACE}%{INT:threads_pending}%{SPACE}%{INT:threads_completed}%{SPACE}%{INT:threads_blocked}%{SPACE}%{INT:threads_all_time_blocked}|%{INT:threads_active}%{SPACE}%{INT:threads_pending})
# WARN [Service Thread] 2016-06-30 14:42:41,538 GCInspector.java:256 - *
CASS_DEFAULT %{CASS_BASE} %{GREEDYDATA:msg}
Raw
filebeat_conf.yaml
filebeat:
spool_size: 500
prospectors:
-
paths:
- "-"
registry_file: /var/tmp/
input_type: stdin
document_type: cassandralog
fields:
timezone: pst
multiline.pattern: '^.*? \[.*\] [0-9]{4}-[0-9]{2}-[0-9]{2}'
multiline.negate: true
multiline.match: after
exclude_lines: ["/system/",
"StatusLogger.java:115",
"StatusLogger.java:51",
"StatusLogger.java:112"
"sstable_activity",
"compaction_history",
"compactions_in_progress",
"local",
"peers",
"size_estimates"]
output:
logstash:
hosts: ["10.0.0.20:5044"]
worker: 1
index: filebeat
logging:
level: debug
# enable file rotation with default configuration
to_files: true
# do not log to syslog
to_syslog: false
files:
path: /var/tmp/
name: mybeat.log
keepfiles: 7
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment