envoy admin stats for issue

stats

cluster.service_primary.bind_errors: 0
cluster.service_primary.external.upstream_rq_200: 7713
cluster.service_primary.external.upstream_rq_2xx: 7713
cluster.service_primary.external.upstream_rq_304: 21
cluster.service_primary.external.upstream_rq_3xx: 21
cluster.service_primary.external.upstream_rq_403: 746
cluster.service_primary.external.upstream_rq_404: 734
cluster.service_primary.external.upstream_rq_4xx: 1480
cluster.service_primary.external.upstream_rq_500: 2573
cluster.service_primary.external.upstream_rq_5xx: 2573
cluster.service_primary.lb_healthy_panic: 0
cluster.service_primary.lb_local_cluster_not_ok: 0
cluster.service_primary.lb_recalculate_zone_structures: 0
cluster.service_primary.lb_subsets_active: 0
cluster.service_primary.lb_subsets_created: 0
cluster.service_primary.lb_subsets_fallback: 0
cluster.service_primary.lb_subsets_removed: 0
cluster.service_primary.lb_subsets_selected: 0
cluster.service_primary.lb_zone_cluster_too_small: 0
cluster.service_primary.lb_zone_no_capacity_left: 0
cluster.service_primary.lb_zone_number_differs: 0
cluster.service_primary.lb_zone_routing_all_directly: 0
cluster.service_primary.lb_zone_routing_cross_zone: 0
cluster.service_primary.lb_zone_routing_sampled: 0
cluster.service_primary.max_host_weight: 0
cluster.service_primary.membership_change: 1
cluster.service_primary.membership_healthy: 1
cluster.service_primary.membership_total: 1
cluster.service_primary.original_dst_host_invalid: 0
cluster.service_primary.retry_or_shadow_abandoned: 0
cluster.service_primary.ssl.ciphers.ECDHE-RSA-AES128-GCM-SHA256: 4829
cluster.service_primary.ssl.connection_error: 0
cluster.service_primary.ssl.fail_verify_cert_hash: 0
cluster.service_primary.ssl.fail_verify_error: 0
cluster.service_primary.ssl.fail_verify_no_cert: 0
cluster.service_primary.ssl.fail_verify_san: 0
cluster.service_primary.ssl.handshake: 4829
cluster.service_primary.ssl.no_certificate: 0
cluster.service_primary.ssl.session_reused: 0
cluster.service_primary.update_attempt: 34301
cluster.service_primary.update_empty: 0
cluster.service_primary.update_failure: 0
cluster.service_primary.update_no_rebuild: 0
cluster.service_primary.update_success: 34301
cluster.service_primary.upstream_cx_active: 1
cluster.service_primary.upstream_cx_close_notify: 0
cluster.service_primary.upstream_cx_connect_attempts_exceeded: 0
cluster.service_primary.upstream_cx_connect_fail: 0
cluster.service_primary.upstream_cx_connect_timeout: 0
cluster.service_primary.upstream_cx_destroy: 0
cluster.service_primary.upstream_cx_destroy_local: 0
cluster.service_primary.upstream_cx_destroy_local_with_active_rq: 338
cluster.service_primary.upstream_cx_destroy_remote: 0
cluster.service_primary.upstream_cx_destroy_remote_with_active_rq: 0
cluster.service_primary.upstream_cx_destroy_with_active_rq: 338
cluster.service_primary.upstream_cx_http1_total: 4829
cluster.service_primary.upstream_cx_http2_total: 0
cluster.service_primary.upstream_cx_idle_timeout: 0
cluster.service_primary.upstream_cx_max_requests: 0
cluster.service_primary.upstream_cx_none_healthy: 0
cluster.service_primary.upstream_cx_overflow: 0
cluster.service_primary.upstream_cx_protocol_error: 0
cluster.service_primary.upstream_cx_rx_bytes_buffered: 13916
cluster.service_primary.upstream_cx_rx_bytes_total: 225928247
cluster.service_primary.upstream_cx_total: 4829
cluster.service_primary.upstream_cx_tx_bytes_buffered: 0
cluster.service_primary.upstream_cx_tx_bytes_total: 12212040
cluster.service_primary.upstream_flow_control_backed_up_total: 0
cluster.service_primary.upstream_flow_control_drained_total: 0
cluster.service_primary.upstream_flow_control_paused_reading_total: 0
cluster.service_primary.upstream_flow_control_resumed_reading_total: 0
cluster.service_primary.upstream_rq_200: 7713
cluster.service_primary.upstream_rq_2xx: 7713
cluster.service_primary.upstream_rq_304: 21
cluster.service_primary.upstream_rq_3xx: 21
cluster.service_primary.upstream_rq_403: 746
cluster.service_primary.upstream_rq_404: 734
cluster.service_primary.upstream_rq_4xx: 1480
cluster.service_primary.upstream_rq_500: 2573
cluster.service_primary.upstream_rq_5xx: 2573
cluster.service_primary.upstream_rq_active: 0
cluster.service_primary.upstream_rq_cancelled: 0
cluster.service_primary.upstream_rq_maintenance_mode: 0
cluster.service_primary.upstream_rq_pending_active: 0
cluster.service_primary.upstream_rq_pending_failure_eject: 0
cluster.service_primary.upstream_rq_pending_overflow: 0
cluster.service_primary.upstream_rq_pending_total: 4829
cluster.service_primary.upstream_rq_per_try_timeout: 0
cluster.service_primary.upstream_rq_retry: 0
cluster.service_primary.upstream_rq_retry_overflow: 0
cluster.service_primary.upstream_rq_retry_success: 0
cluster.service_primary.upstream_rq_rx_reset: 0
cluster.service_primary.upstream_rq_timeout: 0
cluster.service_primary.upstream_rq_total: 12122
cluster.service_primary.upstream_rq_tx_reset: 0
cluster.service_primary.version: 0
cluster.service_shadow.bind_errors: 0
cluster.service_shadow.internal.upstream_rq_200: 7611
cluster.service_shadow.internal.upstream_rq_2xx: 7611
cluster.service_shadow.internal.upstream_rq_403: 753
cluster.service_shadow.internal.upstream_rq_404: 817
cluster.service_shadow.internal.upstream_rq_4xx: 1570
cluster.service_shadow.internal.upstream_rq_500: 2650
cluster.service_shadow.internal.upstream_rq_503: 291
cluster.service_shadow.internal.upstream_rq_5xx: 2941
cluster.service_shadow.lb_healthy_panic: 0
cluster.service_shadow.lb_local_cluster_not_ok: 0
cluster.service_shadow.lb_recalculate_zone_structures: 0
cluster.service_shadow.lb_subsets_active: 0
cluster.service_shadow.lb_subsets_created: 0
cluster.service_shadow.lb_subsets_fallback: 0
cluster.service_shadow.lb_subsets_removed: 0
cluster.service_shadow.lb_subsets_selected: 0
cluster.service_shadow.lb_zone_cluster_too_small: 0
cluster.service_shadow.lb_zone_no_capacity_left: 0
cluster.service_shadow.lb_zone_number_differs: 0
cluster.service_shadow.lb_zone_routing_all_directly: 0
cluster.service_shadow.lb_zone_routing_cross_zone: 0
cluster.service_shadow.lb_zone_routing_sampled: 0
cluster.service_shadow.max_host_weight: 0
cluster.service_shadow.membership_change: 1
cluster.service_shadow.membership_healthy: 1
cluster.service_shadow.membership_total: 1
cluster.service_shadow.original_dst_host_invalid: 0
cluster.service_shadow.retry_or_shadow_abandoned: 0
cluster.service_shadow.update_attempt: 34350
cluster.service_shadow.update_empty: 0
cluster.service_shadow.update_failure: 0
cluster.service_shadow.update_no_rebuild: 0
cluster.service_shadow.update_success: 34350
cluster.service_shadow.upstream_cx_active: 0
cluster.service_shadow.upstream_cx_close_notify: 11831
cluster.service_shadow.upstream_cx_connect_attempts_exceeded: 0
cluster.service_shadow.upstream_cx_connect_fail: 9
cluster.service_shadow.upstream_cx_connect_timeout: 2
cluster.service_shadow.upstream_cx_destroy: 0
cluster.service_shadow.upstream_cx_destroy_local: 0
cluster.service_shadow.upstream_cx_destroy_local_with_active_rq: 0
cluster.service_shadow.upstream_cx_destroy_remote: 0
cluster.service_shadow.upstream_cx_destroy_remote_with_active_rq: 282
cluster.service_shadow.upstream_cx_destroy_with_active_rq: 282
cluster.service_shadow.upstream_cx_http1_total: 12122
cluster.service_shadow.upstream_cx_http2_total: 0
cluster.service_shadow.upstream_cx_idle_timeout: 0
cluster.service_shadow.upstream_cx_max_requests: 0
cluster.service_shadow.upstream_cx_none_healthy: 0
cluster.service_shadow.upstream_cx_overflow: 0
cluster.service_shadow.upstream_cx_protocol_error: 0
cluster.service_shadow.upstream_cx_rx_bytes_buffered: 0
cluster.service_shadow.upstream_cx_rx_bytes_total: 208496928
cluster.service_shadow.upstream_cx_total: 12122
cluster.service_shadow.upstream_cx_tx_bytes_buffered: 0
cluster.service_shadow.upstream_cx_tx_bytes_total: 12759340
cluster.service_shadow.upstream_flow_control_backed_up_total: 0
cluster.service_shadow.upstream_flow_control_drained_total: 0
cluster.service_shadow.upstream_flow_control_paused_reading_total: 0
cluster.service_shadow.upstream_flow_control_resumed_reading_total: 0
cluster.service_shadow.upstream_rq_200: 7611
cluster.service_shadow.upstream_rq_2xx: 7611
cluster.service_shadow.upstream_rq_403: 753
cluster.service_shadow.upstream_rq_404: 817
cluster.service_shadow.upstream_rq_4xx: 1570
cluster.service_shadow.upstream_rq_500: 2650
cluster.service_shadow.upstream_rq_503: 291
cluster.service_shadow.upstream_rq_5xx: 2941
cluster.service_shadow.upstream_rq_active: 0
cluster.service_shadow.upstream_rq_cancelled: 0
cluster.service_shadow.upstream_rq_maintenance_mode: 0
cluster.service_shadow.upstream_rq_pending_active: 0
cluster.service_shadow.upstream_rq_pending_failure_eject: 9
cluster.service_shadow.upstream_rq_pending_overflow: 0
cluster.service_shadow.upstream_rq_pending_total: 12122
cluster.service_shadow.upstream_rq_per_try_timeout: 0
cluster.service_shadow.upstream_rq_retry: 0
cluster.service_shadow.upstream_rq_retry_overflow: 0
cluster.service_shadow.upstream_rq_retry_success: 0
cluster.service_shadow.upstream_rq_rx_reset: 0
cluster.service_shadow.upstream_rq_timeout: 0
cluster.service_shadow.upstream_rq_total: 12113
cluster.service_shadow.upstream_rq_tx_reset: 0
cluster.service_shadow.version: 0
cluster_manager.active_clusters: 2
cluster_manager.cluster_added: 2
cluster_manager.cluster_modified: 0
cluster_manager.cluster_removed: 0
cluster_manager.warming_clusters: 0
filesystem.flushed_by_timer: 17174
filesystem.reopen_failed: 0
filesystem.write_buffered: 23573
filesystem.write_completed: 15454
filesystem.write_total_buffered: 0
http.admin.downstream_cx_active: 2
http.admin.downstream_cx_destroy: 34349
http.admin.downstream_cx_destroy_active_rq: 0
http.admin.downstream_cx_destroy_local: 0
http.admin.downstream_cx_destroy_local_active_rq: 0
http.admin.downstream_cx_destroy_remote: 34349
http.admin.downstream_cx_destroy_remote_active_rq: 0
http.admin.downstream_cx_drain_close: 0
http.admin.downstream_cx_http1_active: 2
http.admin.downstream_cx_http1_total: 2
http.admin.downstream_cx_http2_active: 0
http.admin.downstream_cx_http2_total: 0
http.admin.downstream_cx_idle_timeout: 0
http.admin.downstream_cx_protocol_error: 0
http.admin.downstream_cx_rx_bytes_buffered: 1361
http.admin.downstream_cx_rx_bytes_total: 2394411
http.admin.downstream_cx_ssl_active: 0
http.admin.downstream_cx_ssl_total: 0
http.admin.downstream_cx_total: 34351
http.admin.downstream_cx_tx_bytes_buffered: 0
http.admin.downstream_cx_tx_bytes_total: 410480259
http.admin.downstream_cx_websocket_active: 0
http.admin.downstream_cx_websocket_total: 0
http.admin.downstream_flow_control_paused_reading_total: 0
http.admin.downstream_flow_control_resumed_reading_total: 0
http.admin.downstream_rq_1xx: 0
http.admin.downstream_rq_2xx: 11451
http.admin.downstream_rq_3xx: 0
http.admin.downstream_rq_4xx: 0
http.admin.downstream_rq_5xx: 0
http.admin.downstream_rq_active: 1
http.admin.downstream_rq_http1_total: 11452
http.admin.downstream_rq_http2_total: 0
http.admin.downstream_rq_non_relative_path: 0
http.admin.downstream_rq_response_before_rq_complete: 0
http.admin.downstream_rq_rx_reset: 0
http.admin.downstream_rq_too_large: 0
http.admin.downstream_rq_total: 11452
http.admin.downstream_rq_tx_reset: 0
http.admin.downstream_rq_ws_on_non_ws_route: 0
http.admin.rs_too_large: 0
http.async-client.no_cluster: 0
http.async-client.no_route: 0
http.async-client.rq_direct_response: 0
http.async-client.rq_redirect: 0
http.async-client.rq_total: 12122
http.ingress_http.downstream_cx_active: 0
http.ingress_http.downstream_cx_destroy: 12122
http.ingress_http.downstream_cx_destroy_active_rq: 338
http.ingress_http.downstream_cx_destroy_local: 11781
http.ingress_http.downstream_cx_destroy_local_active_rq: 0
http.ingress_http.downstream_cx_destroy_remote: 341
http.ingress_http.downstream_cx_destroy_remote_active_rq: 338
http.ingress_http.downstream_cx_drain_close: 0
http.ingress_http.downstream_cx_http1_active: 0
http.ingress_http.downstream_cx_http1_total: 12122
http.ingress_http.downstream_cx_http2_active: 0
http.ingress_http.downstream_cx_http2_total: 0
http.ingress_http.downstream_cx_idle_timeout: 0
http.ingress_http.downstream_cx_protocol_error: 0
http.ingress_http.downstream_cx_rx_bytes_buffered: 0
http.ingress_http.downstream_cx_rx_bytes_total: 11734236
http.ingress_http.downstream_cx_ssl_active: 0
http.ingress_http.downstream_cx_ssl_total: 0
http.ingress_http.downstream_cx_total: 12122
http.ingress_http.downstream_cx_tx_bytes_buffered: 0
http.ingress_http.downstream_cx_tx_bytes_total: 226207199
http.ingress_http.downstream_cx_websocket_active: 0
http.ingress_http.downstream_cx_websocket_total: 0
http.ingress_http.downstream_flow_control_paused_reading_total: 0
http.ingress_http.downstream_flow_control_resumed_reading_total: 0
http.ingress_http.downstream_rq_1xx: 0
http.ingress_http.downstream_rq_2xx: 7713
http.ingress_http.downstream_rq_3xx: 21
http.ingress_http.downstream_rq_4xx: 1480
http.ingress_http.downstream_rq_5xx: 2573
http.ingress_http.downstream_rq_active: 0
http.ingress_http.downstream_rq_http1_total: 12122
http.ingress_http.downstream_rq_http2_total: 0
http.ingress_http.downstream_rq_non_relative_path: 0
http.ingress_http.downstream_rq_response_before_rq_complete: 0
http.ingress_http.downstream_rq_rx_reset: 338
http.ingress_http.downstream_rq_too_large: 0
http.ingress_http.downstream_rq_total: 12122
http.ingress_http.downstream_rq_tx_reset: 0
http.ingress_http.downstream_rq_ws_on_non_ws_route: 0
http.ingress_http.no_cluster: 0
http.ingress_http.no_route: 0
http.ingress_http.rq_direct_response: 0
http.ingress_http.rq_redirect: 0
http.ingress_http.rq_total: 12122
http.ingress_http.rs_too_large: 0
http.ingress_http.tracing.client_enabled: 0
http.ingress_http.tracing.health_check: 0
http.ingress_http.tracing.not_traceable: 0
http.ingress_http.tracing.random_sampling: 0
http.ingress_http.tracing.service_forced: 0
listener.0.0.0.0_10000.downstream_cx_active: 0
listener.0.0.0.0_10000.downstream_cx_destroy: 12122
listener.0.0.0.0_10000.downstream_cx_total: 12122
listener.0.0.0.0_10000.http.ingress_http.downstream_rq_1xx: 0
listener.0.0.0.0_10000.http.ingress_http.downstream_rq_2xx: 7713
listener.0.0.0.0_10000.http.ingress_http.downstream_rq_3xx: 21
listener.0.0.0.0_10000.http.ingress_http.downstream_rq_4xx: 1480
listener.0.0.0.0_10000.http.ingress_http.downstream_rq_5xx: 2573
listener.0.0.0.0_10000.no_filter_chain_match: 0
listener.admin.downstream_cx_active: 2
listener.admin.downstream_cx_destroy: 34349
listener.admin.downstream_cx_total: 34351
listener.admin.http.admin.downstream_rq_1xx: 0
listener.admin.http.admin.downstream_rq_2xx: 11451
listener.admin.http.admin.downstream_rq_3xx: 0
listener.admin.http.admin.downstream_rq_4xx: 0
listener.admin.http.admin.downstream_rq_5xx: 0
listener.admin.no_filter_chain_match: 0
listener_manager.listener_added: 1
listener_manager.listener_create_failure: 0
listener_manager.listener_create_success: 4
listener_manager.listener_modified: 0
listener_manager.listener_removed: 0
listener_manager.total_listeners_active: 1
listener_manager.total_listeners_draining: 0
listener_manager.total_listeners_warming: 0
runtime.admin_overrides_active: 0
runtime.load_error: 0
runtime.load_success: 0
runtime.num_keys: 0
runtime.override_dir_exists: 0
runtime.override_dir_not_exists: 0
server.days_until_first_cert_expiring: 2147483647
server.hot_restart_epoch: 0
server.live: 1
server.memory_allocated: 3654064
server.memory_heap_size: 15728640
server.parent_connections: 0
server.total_connections: 0
server.uptime: 171773
server.version: 4151803
server.watchdog_mega_miss: 0
server.watchdog_miss: 0
stats.overflow: 0
cluster.service_primary.external.upstream_rq_time: P0(nan,2) P25(nan,7.05713) P50(nan,839.714) P75(nan,1787.14) P90(nan,4694.17) P95(nan,10541.8) P99(nan,40104) P99.9(nan,55643.2) P100(nan,60000)
cluster.service_primary.upstream_cx_connect_ms: P0(nan,0) P25(nan,3.02929) P50(nan,3.05996) P75(nan,3.09062) P90(nan,4.06154) P95(nan,5.023) P99(nan,9.0805) P99.9(nan,27.171) P100(nan,35)
cluster.service_primary.upstream_cx_length_ms: P0(nan,6000) P25(nan,66025.4) P50(nan,68809.9) P75(nan,105917) P90(nan,153946) P95(nan,198625) P99(nan,317467) P99.9(nan,541720) P100(nan,700000)
cluster.service_primary.upstream_rq_time: P0(nan,2) P25(nan,7.05713) P50(nan,839.714) P75(nan,1787.14) P90(nan,4694.17) P95(nan,10541.8) P99(nan,40104) P99.9(nan,55643.2) P100(nan,60000)
cluster.service_shadow.internal.upstream_rq_time: P0(nan,1) P25(nan,3.06177) P50(nan,1081.43) P75(nan,2208.77) P90(nan,4799.62) P95(nan,7940.31) P99(nan,20158.2) P99.9(nan,40169) P100(nan,140000)
cluster.service_shadow.upstream_cx_connect_ms: P0(nan,0) P25(nan,0) P50(nan,0) P75(nan,0) P90(nan,0) P95(nan,0) P99(nan,1.06599) P99.9(nan,4.07652) P100(nan,9.1)
cluster.service_shadow.upstream_cx_length_ms: P0(nan,0) P25(nan,3.08606) P50(nan,1125.77) P75(nan,2361.3) P90(nan,6131.89) P95(nan,11636.2) P99(nan,31320.8) P99.9(nan,51878) P100(nan,140000)
cluster.service_shadow.upstream_rq_time: P0(nan,1) P25(nan,3.06177) P50(nan,1081.43) P75(nan,2208.77) P90(nan,4799.62) P95(nan,7940.31) P99(nan,20158.2) P99.9(nan,40169) P100(nan,140000)
http.admin.downstream_cx_length_ms: P0(nan,0) P25(nan,0) P50(nan,0) P75(nan,0) P90(nan,0) P95(nan,0) P99(nan,0) P99.9(nan,3.05652) P100(nan,45)
http.admin.downstream_rq_time: P0(0,0) P25(0,0) P50(0,0) P75(0,0) P90(0,0) P95(0,1.0543) P99(0,2.01201) P99.9(0,3.06078) P100(0,11)
http.ingress_http.downstream_cx_length_ms: P0(nan,0) P25(nan,7.08221) P50(nan,878.649) P75(nan,1953.02) P90(nan,6505.33) P95(nan,23223.1) P99(nan,60286.9) P99.9(nan,60928.7) P100(nan,61000)
http.ingress_http.downstream_rq_time: P0(nan,2) P25(nan,7.06406) P50(nan,840.256) P75(nan,1787.14) P90(nan,4694.17) P95(nan,10541.8) P99(nan,40104) P99.9(nan,55643.2) P100(nan,60000)
listener.0.0.0.0_10000.downstream_cx_length_ms: P0(nan,0) P25(nan,7.08116) P50(nan,878.649) P75(nan,1953.02) P90(nan,6505.33) P95(nan,23223.1) P99(nan,60286.9) P99.9(nan,60928.7) P100(nan,61000)
listener.admin.downstream_cx_length_ms: P0(nan,0) P25(nan,0) P50(nan,0) P75(nan,0) P90(nan,0) P95(nan,0) P99(nan,0) P99.9(nan,3.05652) P100(nan,45)