Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save yaauie/dd279e5beda4abbce15ed18fd3673519 to your computer and use it in GitHub Desktop.
Save yaauie/dd279e5beda4abbce15ed18fd3673519 to your computer and use it in GitHub Desktop.

Starting up

One pipeline has high worker_utilization, but hasn't been up for long enough for that to be considered a problem

> GET /_health_report HTTP/1.1
> Host: localhost:9600
> User-Agent: curl/8.7.1
> Accept: */*
> 
< HTTP/1.1 200 OK
< content-type: application/json
< x-content-type-options: nosniff
< Content-Length: 722
< 
{"host":"perhaps","version":"9.0.0","http_address":"127.0.0.1:9600","id":"390c3270-a24d-4f25-8170-e57b3a563dff","name":"perhaps","ephemeral_id":"8e67ae0b-e074-4402-8a41-f99eefaa695f","snapshot":null,"status":"green","symptom":"1 indicator is healthy (`pipelines`)","indicators":{"pipelines":{"status":"green","symptom":"2 indicators are healthy (`blocker`, `anchor`)","indicators":{"anchor":{"status":"green","symptom":"The pipeline is healthy","details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":0.0,"lifetime":0.0}}}},"blocker":{"status":"green","symptom":"The pipeline is healthy","details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":88.72,"lifetime":88.72}}}}}}}}
{
  "host": "perhaps",
  "version": "9.0.0",
  "http_address": "127.0.0.1:9600",
  "id": "390c3270-a24d-4f25-8170-e57b3a563dff",
  "name": "perhaps",
  "ephemeral_id": "8e67ae0b-e074-4402-8a41-f99eefaa695f",
  "snapshot": null,
  "status": "green",
  "symptom": "1 indicator is healthy (`pipelines`)",
  "indicators": {
    "pipelines": {
      "status": "green",
      "symptom": "2 indicators are healthy (`blocker`, `anchor`)",
      "indicators": {
        "anchor": {
          "status": "green",
          "symptom": "The pipeline is healthy",
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 0.0,
                "lifetime": 0.0
              }
            }
          }
        },
        "blocker": {
          "status": "green",
          "symptom": "The pipeline is healthy",
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 88.72,
                "lifetime": 88.72
              }
            }
          }
        }
      }
    }
  }
}

Concerning (Nearly Blocked One Minute)

Our blocker pipeline has a flow.worker_utilization.last_1_minute > 95

> GET /_health_report HTTP/1.1
> Host: localhost:9600
> User-Agent: curl/8.7.1
> Accept: */*
> 
< HTTP/1.1 200 OK
< content-type: application/json
< x-content-type-options: nosniff
< Content-Length: 1355
< 
{"host":"perhaps","version":"9.0.0","http_address":"127.0.0.1:9600","id":"390c3270-a24d-4f25-8170-e57b3a563dff","name":"perhaps","ephemeral_id":"8e67ae0b-e074-4402-8a41-f99eefaa695f","snapshot":null,"status":"yellow","symptom":"1 indicator is concerning (`pipelines`)","indicators":{"pipelines":{"status":"yellow","symptom":"1 indicator is concerning (`blocker`) and 1 indicator is healthy (`anchor`)","indicators":{"anchor":{"status":"green","symptom":"The pipeline is healthy","details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":0.0,"lifetime":2.82E-4}}}},"blocker":{"status":"yellow","symptom":"The pipeline is concerning; 1 area is impacted and 1 diagnosis is available","diagnosis":[{"id":"logstash:health:pipeline:flow:worker_utilization:diagnosis:1m-nearly-blocked","cause":"pipeline workers have been nearly blocked for at least one minute","action":"address bottleneck or add resources","help_url":"https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#nearly-blocked-1m"}],"impacts":[{"id":"logstash:health:pipeline:flow:impact:blocked_processing","severity":2,"description":"the pipeline is blocked","impact_areas":["pipeline_execution"]}],"details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"lifetime":99.6,"current":100.0,"last_1_minute":99.6}}}}}}}}
{
  "host": "perhaps",
  "version": "9.0.0",
  "http_address": "127.0.0.1:9600",
  "id": "390c3270-a24d-4f25-8170-e57b3a563dff",
  "name": "perhaps",
  "ephemeral_id": "8e67ae0b-e074-4402-8a41-f99eefaa695f",
  "snapshot": null,
  "status": "yellow",
  "symptom": "1 indicator is concerning (`pipelines`)",
  "indicators": {
    "pipelines": {
      "status": "yellow",
      "symptom": "1 indicator is concerning (`blocker`) and 1 indicator is healthy (`anchor`)",
      "indicators": {
        "anchor": {
          "status": "green",
          "symptom": "The pipeline is healthy",
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 0.0,
                "lifetime": 0.000282
              }
            }
          }
        },
        "blocker": {
          "status": "yellow",
          "symptom": "The pipeline is concerning; 1 area is impacted and 1 diagnosis is available",
          "diagnosis": [
            {
              "id": "logstash:health:pipeline:flow:worker_utilization:diagnosis:1m-nearly-blocked",
              "cause": "pipeline workers have been nearly blocked for at least one minute",
              "action": "address bottleneck or add resources",
              "help_url": "https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#nearly-blocked-1m"
            }
          ],
          "impacts": [
            {
              "id": "logstash:health:pipeline:flow:impact:blocked_processing",
              "severity": 2,
              "description": "the pipeline is blocked",
              "impact_areas": [
                "pipeline_execution"
              ]
            }
          ],
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "lifetime": 99.6,
                "current": 100.0,
                "last_1_minute": 99.6
              }
            }
          }
        }
      }
    }
  }
}

Concerning (Blocked One Minute)

Our blocker pipeline has a flow.worker_utilization.last_1_minute >= 99.999

> GET /_health_report HTTP/1.1
> Host: localhost:9600
> User-Agent: curl/8.7.1
> Accept: */*
> 
< HTTP/1.1 200 OK
< content-type: application/json
< x-content-type-options: nosniff
< Content-Length: 1378
< 
{"host":"perhaps","version":"9.0.0","http_address":"127.0.0.1:9600","id":"390c3270-a24d-4f25-8170-e57b3a563dff","name":"perhaps","ephemeral_id":"8e67ae0b-e074-4402-8a41-f99eefaa695f","snapshot":null,"status":"yellow","symptom":"1 indicator is concerning (`pipelines`)","indicators":{"pipelines":{"status":"yellow","symptom":"1 indicator is concerning (`blocker`) and 1 indicator is healthy (`anchor`)","indicators":{"anchor":{"status":"green","symptom":"The pipeline is healthy","details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"lifetime":3.909E-4,"current":6.442E-4,"last_1_minute":3.968E-4}}}},"blocker":{"status":"yellow","symptom":"The pipeline is concerning; 1 area is impacted and 1 diagnosis is available","diagnosis":[{"id":"logstash:health:pipeline:flow:worker_utilization:diagnosis:1m-blocked","cause":"pipeline workers have been completely blocked for at least one minute","action":"address bottleneck or add resources","help_url":"https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#blocked-1m"}],"impacts":[{"id":"logstash:health:pipeline:flow:impact:blocked_processing","severity":2,"description":"the pipeline is blocked","impact_areas":["pipeline_execution"]}],"details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"lifetime":99.63,"current":100.0,"last_1_minute":100.0}}}}}}}}
{
  "host": "perhaps",
  "version": "9.0.0",
  "http_address": "127.0.0.1:9600",
  "id": "390c3270-a24d-4f25-8170-e57b3a563dff",
  "name": "perhaps",
  "ephemeral_id": "8e67ae0b-e074-4402-8a41-f99eefaa695f",
  "snapshot": null,
  "status": "yellow",
  "symptom": "1 indicator is concerning (`pipelines`)",
  "indicators": {
    "pipelines": {
      "status": "yellow",
      "symptom": "1 indicator is concerning (`blocker`) and 1 indicator is healthy (`anchor`)",
      "indicators": {
        "anchor": {
          "status": "green",
          "symptom": "The pipeline is healthy",
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "lifetime": 0.0003909,
                "current": 0.0006442,
                "last_1_minute": 0.0003968
              }
            }
          }
        },
        "blocker": {
          "status": "yellow",
          "symptom": "The pipeline is concerning; 1 area is impacted and 1 diagnosis is available",
          "diagnosis": [
            {
              "id": "logstash:health:pipeline:flow:worker_utilization:diagnosis:1m-blocked",
              "cause": "pipeline workers have been completely blocked for at least one minute",
              "action": "address bottleneck or add resources",
              "help_url": "https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#blocked-1m"
            }
          ],
          "impacts": [
            {
              "id": "logstash:health:pipeline:flow:impact:blocked_processing",
              "severity": 2,
              "description": "the pipeline is blocked",
              "impact_areas": [
                "pipeline_execution"
              ]
            }
          ],
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "lifetime": 99.63,
                "current": 100.0,
                "last_1_minute": 100.0
              }
            }
          }
        }
      }
    }
  }
}

Concerning (Nearly Blocked Five Minutes)

Our blocker pipeline has a flow.worker_utilization.last_5_minutes > 95

> GET /_health_report HTTP/1.1
> Host: localhost:9600
> User-Agent: curl/8.7.1
> Accept: */*
> 
< HTTP/1.1 200 OK
< content-type: application/json
< x-content-type-options: nosniff
< Content-Length: 1407
< 
{"host":"perhaps","version":"9.0.0","http_address":"127.0.0.1:9600","id":"390c3270-a24d-4f25-8170-e57b3a563dff","name":"perhaps","ephemeral_id":"8e67ae0b-e074-4402-8a41-f99eefaa695f","snapshot":null,"status":"yellow","symptom":"1 indicator is concerning (`pipelines`)","indicators":{"pipelines":{"status":"yellow","symptom":"1 indicator is concerning (`blocker`) and 1 indicator is healthy (`anchor`)","indicators":{"anchor":{"status":"green","symptom":"The pipeline is healthy","details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"lifetime":3.066E-4,"current":0.0,"last_1_minute":2.66E-4}}}},"blocker":{"status":"yellow","symptom":"The pipeline is concerning; 1 area is impacted and 1 diagnosis is available","diagnosis":[{"id":"logstash:health:pipeline:flow:worker_utilization:diagnosis:5m-nearly-blocked","cause":"pipeline workers have been nearly blocked for at least five minutes","action":"address bottleneck or add resources","help_url":"https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#nearly-blocked-5m"}],"impacts":[{"id":"logstash:health:pipeline:flow:impact:blocked_processing","severity":1,"description":"the pipeline is blocked","impact_areas":["pipeline_execution"]}],"details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":100.0,"lifetime":99.92,"last_1_minute":100.0,"last_5_minutes":99.92}}}}}}}}
{
  "host": "perhaps",
  "version": "9.0.0",
  "http_address": "127.0.0.1:9600",
  "id": "390c3270-a24d-4f25-8170-e57b3a563dff",
  "name": "perhaps",
  "ephemeral_id": "8e67ae0b-e074-4402-8a41-f99eefaa695f",
  "snapshot": null,
  "status": "yellow",
  "symptom": "1 indicator is concerning (`pipelines`)",
  "indicators": {
    "pipelines": {
      "status": "yellow",
      "symptom": "1 indicator is concerning (`blocker`) and 1 indicator is healthy (`anchor`)",
      "indicators": {
        "anchor": {
          "status": "green",
          "symptom": "The pipeline is healthy",
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "lifetime": 0.0003066,
                "current": 0.0,
                "last_1_minute": 0.000266
              }
            }
          }
        },
        "blocker": {
          "status": "yellow",
          "symptom": "The pipeline is concerning; 1 area is impacted and 1 diagnosis is available",
          "diagnosis": [
            {
              "id": "logstash:health:pipeline:flow:worker_utilization:diagnosis:5m-nearly-blocked",
              "cause": "pipeline workers have been nearly blocked for at least five minutes",
              "action": "address bottleneck or add resources",
              "help_url": "https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#nearly-blocked-5m"
            }
          ],
          "impacts": [
            {
              "id": "logstash:health:pipeline:flow:impact:blocked_processing",
              "severity": 1,
              "description": "the pipeline is blocked",
              "impact_areas": [
                "pipeline_execution"
              ]
            }
          ],
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 100.0,
                "lifetime": 99.92,
                "last_1_minute": 100.0,
                "last_5_minutes": 99.92
              }
            }
          }
        }
      }
    }
  }
}

Unhealthy (Blocked Five Minutes)

Our blocker pipeline has a flow.worker_utilization.last_5_minutes >= 99.999

> GET /_health_report HTTP/1.1
> Host: localhost:9600
> User-Agent: curl/8.7.1
> Accept: */*
> 
< HTTP/1.1 200 OK
< content-type: application/json
< x-content-type-options: nosniff
< Content-Length: 1417
< 
{"host":"perhaps","version":"9.0.0","http_address":"127.0.0.1:9600","id":"390c3270-a24d-4f25-8170-e57b3a563dff","name":"perhaps","ephemeral_id":"8e67ae0b-e074-4402-8a41-f99eefaa695f","snapshot":null,"status":"red","symptom":"1 indicator is unhealthy (`pipelines`)","indicators":{"pipelines":{"status":"red","symptom":"1 indicator is unhealthy (`blocker`) and 1 indicator is healthy (`anchor`)","indicators":{"anchor":{"status":"green","symptom":"The pipeline is healthy","details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":5.894E-4,"lifetime":3.169E-4,"last_1_minute":2.596E-4,"last_5_minutes":3.283E-4}}}},"blocker":{"status":"red","symptom":"The pipeline is unhealthy; 1 area is impacted and 1 diagnosis is available","diagnosis":[{"id":"logstash:health:pipeline:flow:worker_utilization:diagnosis:5m-blocked","cause":"pipeline workers have been completely blocked for at least five minutes","action":"address bottleneck or add resources","help_url":"https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#blocked-5m"}],"impacts":[{"id":"logstash:health:pipeline:flow:impact:blocked_processing","severity":1,"description":"the pipeline is blocked","impact_areas":["pipeline_execution"]}],"details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":100.0,"lifetime":99.92,"last_1_minute":100.0,"last_5_minutes":100.0}}}}}}}}
{
  "host": "perhaps",
  "version": "9.0.0",
  "http_address": "127.0.0.1:9600",
  "id": "390c3270-a24d-4f25-8170-e57b3a563dff",
  "name": "perhaps",
  "ephemeral_id": "8e67ae0b-e074-4402-8a41-f99eefaa695f",
  "snapshot": null,
  "status": "red",
  "symptom": "1 indicator is unhealthy (`pipelines`)",
  "indicators": {
    "pipelines": {
      "status": "red",
      "symptom": "1 indicator is unhealthy (`blocker`) and 1 indicator is healthy (`anchor`)",
      "indicators": {
        "anchor": {
          "status": "green",
          "symptom": "The pipeline is healthy",
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 0.0005894,
                "lifetime": 0.0003169,
                "last_1_minute": 0.0002596,
                "last_5_minutes": 0.0003283
              }
            }
          }
        },
        "blocker": {
          "status": "red",
          "symptom": "The pipeline is unhealthy; 1 area is impacted and 1 diagnosis is available",
          "diagnosis": [
            {
              "id": "logstash:health:pipeline:flow:worker_utilization:diagnosis:5m-blocked",
              "cause": "pipeline workers have been completely blocked for at least five minutes",
              "action": "address bottleneck or add resources",
              "help_url": "https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#blocked-5m"
            }
          ],
          "impacts": [
            {
              "id": "logstash:health:pipeline:flow:impact:blocked_processing",
              "severity": 1,
              "description": "the pipeline is blocked",
              "impact_areas": [
                "pipeline_execution"
              ]
            }
          ],
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 100.0,
                "lifetime": 99.92,
                "last_1_minute": 100.0,
                "last_5_minutes": 100.0
              }
            }
          }
        }
      }
    }
  }
}

Concerning (Nearly Blocked Five Minutes)

Our blocker pipeline has a flow.worker_utilization.last_5_minutes > 95, and we have not yet detected that a recovery is in process because its flow.worker_utilization.last_1_minute is still > 80

> GET /_health_report HTTP/1.1
> Host: localhost:9600
> User-Agent: curl/8.7.1
> Accept: */*
> 
< HTTP/1.1 200 OK
< content-type: application/json
< x-content-type-options: nosniff
< Content-Length: 1432
< 
{"host":"perhaps","version":"9.0.0","http_address":"127.0.0.1:9600","id":"390c3270-a24d-4f25-8170-e57b3a563dff","name":"perhaps","ephemeral_id":"8e67ae0b-e074-4402-8a41-f99eefaa695f","snapshot":null,"status":"yellow","symptom":"1 indicator is concerning (`pipelines`)","indicators":{"pipelines":{"status":"yellow","symptom":"1 indicator is concerning (`blocker`) and 1 indicator is healthy (`anchor`)","indicators":{"anchor":{"status":"green","symptom":"The pipeline is healthy","details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":0.0,"lifetime":2.987E-4,"last_1_minute":2.63E-4,"last_5_minutes":2.743E-4}}}},"blocker":{"status":"yellow","symptom":"The pipeline is concerning; 1 area is impacted and 1 diagnosis is available","diagnosis":[{"id":"logstash:health:pipeline:flow:worker_utilization:diagnosis:5m-nearly-blocked","cause":"pipeline workers have been nearly blocked for at least five minutes","action":"address bottleneck or add resources","help_url":"https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#nearly-blocked-5m"}],"impacts":[{"id":"logstash:health:pipeline:flow:impact:blocked_processing","severity":1,"description":"the pipeline is blocked","impact_areas":["pipeline_execution"]}],"details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":90.01,"lifetime":99.53,"last_1_minute":97.9,"last_5_minutes":99.56}}}}}}}}
{
  "host": "perhaps",
  "version": "9.0.0",
  "http_address": "127.0.0.1:9600",
  "id": "390c3270-a24d-4f25-8170-e57b3a563dff",
  "name": "perhaps",
  "ephemeral_id": "8e67ae0b-e074-4402-8a41-f99eefaa695f",
  "snapshot": null,
  "status": "yellow",
  "symptom": "1 indicator is concerning (`pipelines`)",
  "indicators": {
    "pipelines": {
      "status": "yellow",
      "symptom": "1 indicator is concerning (`blocker`) and 1 indicator is healthy (`anchor`)",
      "indicators": {
        "anchor": {
          "status": "green",
          "symptom": "The pipeline is healthy",
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 0.0,
                "lifetime": 0.0002987,
                "last_1_minute": 0.000263,
                "last_5_minutes": 0.0002743
              }
            }
          }
        },
        "blocker": {
          "status": "yellow",
          "symptom": "The pipeline is concerning; 1 area is impacted and 1 diagnosis is available",
          "diagnosis": [
            {
              "id": "logstash:health:pipeline:flow:worker_utilization:diagnosis:5m-nearly-blocked",
              "cause": "pipeline workers have been nearly blocked for at least five minutes",
              "action": "address bottleneck or add resources",
              "help_url": "https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#nearly-blocked-5m"
            }
          ],
          "impacts": [
            {
              "id": "logstash:health:pipeline:flow:impact:blocked_processing",
              "severity": 1,
              "description": "the pipeline is blocked",
              "impact_areas": [
                "pipeline_execution"
              ]
            }
          ],
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 90.01,
                "lifetime": 99.53,
                "last_1_minute": 97.9,
                "last_5_minutes": 99.56
              }
            }
          }
        }
      }
    }
  }
}

Concerning (Nearly Blocked Five Minutes, Recovery Detected)

Our blocker pipeline has a flow.worker_utilization.last_5_minutes > 95, and we have detected that a recovery is in process because its flow.worker_utilization.last_1_minute is <= 80

> GET /_health_report HTTP/1.1
> Host: localhost:9600
> User-Agent: curl/8.7.1
> Accept: */*
> 
< HTTP/1.1 200 OK
< content-type: application/json
< x-content-type-options: nosniff
< Content-Length: 1449
< 
{"host":"perhaps","version":"9.0.0","http_address":"127.0.0.1:9600","id":"390c3270-a24d-4f25-8170-e57b3a563dff","name":"perhaps","ephemeral_id":"8e67ae0b-e074-4402-8a41-f99eefaa695f","snapshot":null,"status":"yellow","symptom":"1 indicator is concerning (`pipelines`)","indicators":{"pipelines":{"status":"yellow","symptom":"1 indicator is concerning (`blocker`) and 1 indicator is healthy (`anchor`)","indicators":{"anchor":{"status":"green","symptom":"The pipeline is healthy","details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":0.0,"lifetime":2.873E-4,"last_1_minute":1.351E-4,"last_5_minutes":2.714E-4}}}},"blocker":{"status":"yellow","symptom":"The pipeline is concerning; 1 area is impacted and 1 diagnosis is available","diagnosis":[{"id":"logstash:health:pipeline:flow:worker_utilization:diagnosis:5m-nearly-blocked","cause":"pipeline workers have been nearly blocked for at least five minutes, but they appear to be recovering","action":"continue to monitor","help_url":"https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#nearly-blocked-5m"}],"impacts":[{"id":"logstash:health:pipeline:flow:impact:blocked_processing","severity":1,"description":"the pipeline is blocked","impact_areas":["pipeline_execution"]}],"details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":3.847,"lifetime":95.9,"last_1_minute":77.1,"last_5_minutes":95.4}}}}}}}}
{
  "host": "perhaps",
  "version": "9.0.0",
  "http_address": "127.0.0.1:9600",
  "id": "390c3270-a24d-4f25-8170-e57b3a563dff",
  "name": "perhaps",
  "ephemeral_id": "8e67ae0b-e074-4402-8a41-f99eefaa695f",
  "snapshot": null,
  "status": "yellow",
  "symptom": "1 indicator is concerning (`pipelines`)",
  "indicators": {
    "pipelines": {
      "status": "yellow",
      "symptom": "1 indicator is concerning (`blocker`) and 1 indicator is healthy (`anchor`)",
      "indicators": {
        "anchor": {
          "status": "green",
          "symptom": "The pipeline is healthy",
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 0.0,
                "lifetime": 0.0002873,
                "last_1_minute": 0.0001351,
                "last_5_minutes": 0.0002714
              }
            }
          }
        },
        "blocker": {
          "status": "yellow",
          "symptom": "The pipeline is concerning; 1 area is impacted and 1 diagnosis is available",
          "diagnosis": [
            {
              "id": "logstash:health:pipeline:flow:worker_utilization:diagnosis:5m-nearly-blocked",
              "cause": "pipeline workers have been nearly blocked for at least five minutes, but they appear to be recovering",
              "action": "continue to monitor",
              "help_url": "https://www.elastic.co/guide/en/logstash/master/health-report-pipeline-flow-worker-utilization.html#nearly-blocked-5m"
            }
          ],
          "impacts": [
            {
              "id": "logstash:health:pipeline:flow:impact:blocked_processing",
              "severity": 1,
              "description": "the pipeline is blocked",
              "impact_areas": [
                "pipeline_execution"
              ]
            }
          ],
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 3.847,
                "lifetime": 95.9,
                "last_1_minute": 77.1,
                "last_5_minutes": 95.4
              }
            }
          }
        }
      }
    }
  }
}

Healthy

Our blocker pipeline's worker_utilization flows are all under relevant thresholds:

> GET /_health_report HTTP/1.1
> Host: localhost:9600
> User-Agent: curl/8.7.1
> Accept: */*
> 
< HTTP/1.1 200 OK
< content-type: application/json
< x-content-type-options: nosniff
< Content-Length: 823
< 
{"host":"perhaps","version":"9.0.0","http_address":"127.0.0.1:9600","id":"390c3270-a24d-4f25-8170-e57b3a563dff","name":"perhaps","ephemeral_id":"8e67ae0b-e074-4402-8a41-f99eefaa695f","snapshot":null,"status":"green","symptom":"1 indicator is healthy (`pipelines`)","indicators":{"pipelines":{"status":"green","symptom":"2 indicators are healthy (`blocker`, `anchor`)","indicators":{"anchor":{"status":"green","symptom":"The pipeline is healthy","details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":0.0,"lifetime":2.831E-4,"last_1_minute":1.348E-4,"last_5_minutes":2.758E-4}}}},"blocker":{"status":"green","symptom":"The pipeline is healthy","details":{"status":{"state":"RUNNING"},"flow":{"worker_utilization":{"current":3.816,"lifetime":94.56,"last_1_minute":69.13,"last_5_minutes":93.68}}}}}}}}
{
  "host": "perhaps",
  "version": "9.0.0",
  "http_address": "127.0.0.1:9600",
  "id": "390c3270-a24d-4f25-8170-e57b3a563dff",
  "name": "perhaps",
  "ephemeral_id": "8e67ae0b-e074-4402-8a41-f99eefaa695f",
  "snapshot": null,
  "status": "green",
  "symptom": "1 indicator is healthy (`pipelines`)",
  "indicators": {
    "pipelines": {
      "status": "green",
      "symptom": "2 indicators are healthy (`blocker`, `anchor`)",
      "indicators": {
        "anchor": {
          "status": "green",
          "symptom": "The pipeline is healthy",
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 0.0,
                "lifetime": 0.0002831,
                "last_1_minute": 0.0001348,
                "last_5_minutes": 0.0002758
              }
            }
          }
        },
        "blocker": {
          "status": "green",
          "symptom": "The pipeline is healthy",
          "details": {
            "status": {
              "state": "RUNNING"
            },
            "flow": {
              "worker_utilization": {
                "current": 3.816,
                "lifetime": 94.56,
                "last_1_minute": 69.13,
                "last_5_minutes": 93.68
              }
            }
          }
        }
      }
    }
  }
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment