ebpf_exporter configuration with a program to measure audit_log_start latency

ebpf-exporter-config.yaml

programs:
  - name: audit-log-start-latency
    metrics:
      histograms:
        - name: function_latency_seconds
          help: Latency of Linux kernel function
          table: dist
          bucket_type: fixed
          bucket_keys:
            - 1000     # 1ms
            - 10000    # 10ms
            - 100000   # 100ms
            - 1000000  # 1s
            - 10000000 # 10s
          bucket_multiplier: 1e-06 # microseconds to seconds
          labels:
            - name: function
              size: 8
              decoders:
                - name: ksym
            - name: bucket
              size: 8
              decoders:
                - name: uint
    kprobes:
      audit_log_start: trace_func_entry
    kretprobes:
      audit_log_start: trace_func_return
    code: |
      #include <uapi/linux/ptrace.h>

      // Set up user-defined buckets.
      const static u64 buckets[] = {
          1000,    // 1ms
          10000,   // 10ms
          100000,  // 100ms
          1000000, // 1s
          10000000 // 10s
      };
      const static int NUM_BUCKETS = sizeof(buckets) / sizeof(buckets[0]);

      // Define the key used in the latency histogram.
      typedef struct hist_key {
          u64 ip;
          u64 bucket;
      } hist_key_t;

      // Used to keep track of the start time of kernel routines.
      BPF_HASH(start, u32);
      // Used to keep track of the address of the kernel routine kprobe.
      BPF_HASH(ipaddr, u32);
      // Used to record the latency of kernel routines.
      BPF_HISTOGRAM(dist, hist_key_t);

      static u64 get_bucket(u64 value) {
          for (int i = 0; i < NUM_BUCKETS; i++) {
              if (value <= buckets[i]) {
                  return buckets[i];
              }
          }
          // Cap at maximum value
          return buckets[NUM_BUCKETS - 1];
      }

      // Called when a kprobe is entered.
      int trace_func_entry(struct pt_regs *ctx)
      {
          // Get the process ID that resulted in this kprobe.
          u64 pid_tgid = bpf_get_current_pid_tgid();
          u32 pid = pid_tgid;

          // Get and record the current kernel time (in nanoseconds).
          u64 ts = bpf_ktime_get_ns();
          start.update(&pid, &ts);

          // Get and record the kprobe address.
          u64 ip = PT_REGS_IP(ctx);
          ipaddr.update(&pid, &ip);

          return 0;
      }

      // Called when a kprobe returns.
      int trace_func_return(struct pt_regs *ctx)
      {
          // Get the process ID that resulted in this kprobe.
          u64 pid_tgid = bpf_get_current_pid_tgid();
          u32 pid = pid_tgid;

          // Will be used to retrieve start time and calculate latency.
          u64 *tsp, delta;

          // Retrieve the start time that was stored in trace_func_entry.
          tsp = start.lookup(&pid);
          // Return is no start time was found.
          if (tsp == 0) {
              return 0;
          }
          // Calculate the latency of the .
          delta = bpf_ktime_get_ns() - *tsp;
          // Convert nanoseconds to microseconds.
          delta /= 1000;
          // Remove the start time from the hash.
          start.delete(&pid);

          // Retrieve the kprobe address.
          u64 ip, *ipp = ipaddr.lookup(&pid);
          // Return if kprobe address not found.
          if (ipp == 0) {
              return 0;
          }
          ip = *ipp;

          // Construct histogram key.
          hist_key_t key;
          // From ebpf_exporter docs:
          // > Note that sometimes you can observe PT_REGS_IP being off by one.
          // > You can subtract 1 in your code to make it point to the right
          // > instruction that can be found /proc/kallsyms.
          key.ip = ip - 1;

          // Get the user-defined bucket of the latency, and increment the
          // slot for that value in the latency histogram.
          key.bucket = get_bucket(delta);
          dist.increment(key);

          // Increment the optional sum key.
          hist_key_t max_key;
          max_key.bucket = buckets[NUM_BUCKETS - 1] + 1;
          max_key.ip = key.ip;
          dist.increment(max_key, delta);

          // Delete the kprobe address from the hash.
          ipaddr.delete(&pid);

          return 0;
      }