prometheus-exclude-docker-and-non-ext-fs.yaml

prometheus-exclude-docker-and-non-ext-fs.yaml

# Purpose: Clean up non-sense filesystem entries from node exporter's metrics
#
#  * Remove all /var/lib/docker, /var/run, /run mount points
#  * Remove all non-ext filesystem
#
# Author : Ky-Anh Huynh
# License: as free as beer, or a MIT
# Efficiency : 108724 metrics --> 1593 metrics

# TODO: support xfs filesystem

# Note you may want to enable prometheus Admin APi and clean up all known metrics
# curl -X POST -g 'http:/prometheus.lauxanh.us/api/v1/admin/tsdb/delete_series?match[]={__name__=~"node_filesystem.*"}'

# Node exporter generates too many metrics

node_filesystem_avail{filesystem!~"^/var/lib/docker.+|^/var/run/.+|^/run.*"}
node_filesystem_free{filesystem!~"^/var/lib/docker.+|^/var/run/.+|^/run.*"}

node_filesystem_files{fstype=~"ext.+|^$",filesystem!~"^/var/lib/docker.+|^/var/run/.+|^/run.*"}
node_filesystem_files_free{fstype=~"ext.+|^$",filesystem!~"^/var/lib/docker.+|^/var/run/.+|^/run.*"}

# Let's see

node_network_transmit_packets{}                                              7170
node_filesystem_avail{}                                                      21406
node_filesystem_free{}                                                       21406
node_filesystem_size{}                                                       21406
node_filesystem_files{}                                                      21648
node_filesystem_files_free{}                                                 21648
node_systemd_unit_state{}                                                    33220

# Now clean up

  metric_relabel_configs:

  - source_labels:  [__name__]
    regex:          ^node_systemd_unit_state
    action:         drop

  - source_labels:  [__name__, filesystem]
    regex:          "^node_filesystem.*;(/var/lib/docker.+|/var/run/.+|/run.*)"
    action:         drop

  - source_labels:  [__name__, fstype]
    regex:          "^(node_filesystem_size_bytes|node_filesystem_files).*;.+"
    action:         replace
    target_label:   __ops_tmp
    replacement:    removeme

  - source_labels:  [__name__, fstype]
    regex:          "^(node_filesystem_size_bytes|node_filesystem_files).*;ext.+"
    action:         replace
    target_label:   __ops_tmp
    replacement:    keepme

  - source_labels:  [__name__, __ops_tmp]
    regex:          "^(node_filesystem_size_bytes|node_filesystem_files).*;removeme"
    action:         drop

## And let's see the result

node_filesystem_avail{}                                                      65
node_filesystem_free{}                                                       65
node_filesystem_size{}                                                       65
node_filesystem_files{}                                                      94
node_filesystem_files_free{}                                                 94
node_filesystem_avail_bytes{}                                                242
node_filesystem_device_error{}                                               242
node_filesystem_free_bytes{}                                                 242
node_filesystem_readonly{}                                                   242
node_filesystem_size_bytes{}                                                 242

Thanks bro for the tip.