docker.jsonnet

{
  _config+:: {
    hako_docker_alerts: {
      HighMemoryUtilizationThreshold: '95',
    },
  },
  prometheusAlerts+:: {
    groups+: [
      {
        name: 'hako-docker',
        rules: [
          {
            alert: 'ContainerMemoryUtilizationTooHigh',
            expr: |||
              avg (
                  container_memory_rss / (container_spec_memory_limit_bytes > 0)
                  )
                by (
                    container_label_com_amazonaws_ecs_container_name,
                    container_label_com_amazonaws_ecs_task_definition_family,
                    container_label_com_amazonaws_ecs_cluster
                    )
                * 100
                > %(HighMemoryUtilizationThreshold)s
            ||| % $._config.hako_docker_alerts,
            'for': '5m',
            labels: {
              pager: 'hako-alert',
              severity: 'warning',
            },
            annotations: {
              title: '{{ $labels.container_label_com_amazonaws_ecs_task_definition_family }} ({{ $labels.container_label_com_amazonaws_ecs_container_name }}) RSS Memory utilization alert',
              description: '{{ $labels.container_label_com_amazonaws_ecs_task_definition_family }} ({{ $labels.container_label_com_amazonaws_ecs_container_name }}) RSS memory utilization over %(HighMemoryUtilizationThreshold)s percent' % $._config.hako_docker_alerts,
              runbook: 'https://docs.example.com/pages/runbook/ContainerMemoryUtilizationTooHigh',
            },
          },
        ],
      },
    ],
  },
}