hanging prometheus

dashboard

http://imgur.com/a/jQb2y

flags

Prometheus
Alerts
Graph
Status 
Help
Command-Line Flags
alertmanager.notification-queue-capacity	10000
alertmanager.timeout	10s
alertmanager.url	
config.file	/etc/prometheus/prometheus.yaml
log.format	"logger:stderr"
log.level	"info"
query.max-concurrency	20
query.staleness-delta	5m0s
query.timeout	2m0s
storage.local.checkpoint-dirty-series-limit	5000
storage.local.checkpoint-interval	5m0s
storage.local.chunk-encoding-version	1
storage.local.dirty	false
storage.local.engine	persisted
storage.local.index-cache-size.fingerprint-to-metric	10485760
storage.local.index-cache-size.fingerprint-to-timerange	5242880
storage.local.index-cache-size.label-name-to-label-values	10485760
storage.local.index-cache-size.label-pair-to-fingerprints	20971520
storage.local.max-chunks-to-persist	2097152
storage.local.memory-chunks	4194304
storage.local.num-fingerprint-mutexes	4096
storage.local.path	/prometheus
storage.local.pedantic-checks	false
storage.local.retention	168h0m0s
storage.local.series-file-shrink-ratio	0.1
storage.local.series-sync-strategy	adaptive
storage.remote.graphite-address	
storage.remote.graphite-prefix	
storage.remote.graphite-transport	tcp
storage.remote.influxdb-url	
storage.remote.influxdb.database	prometheus
storage.remote.influxdb.retention-policy	default
storage.remote.influxdb.username	
storage.remote.opentsdb-url	
storage.remote.timeout	30s
version	false
web.console.libraries	/etc/prometheus/console_libraries
web.console.templates	/etc/prometheus/consoles
web.enable-remote-shutdown	false
web.external-url	https://prometheus.staging.cloud.sap
web.listen-address	:9090
web.max-connections	512
web.read-timeout	30s
web.route-prefix	/
web.telemetry-path	/metrics
web.user-assets	

goroutine

goroutine 7895394 [running]:
runtime/pprof.writeGoroutineStacks(0x2683540, 0xc9cfcd15f0, 0x19424c0, 0x30)
	/usr/local/go/src/runtime/pprof/pprof.go:585 +0x79
runtime/pprof.writeGoroutine(0x2683540, 0xc9cfcd15f0, 0x2, 0x0, 0x1a2ade0)
	/usr/local/go/src/runtime/pprof/pprof.go:574 +0x44
runtime/pprof.(*Profile).WriteTo(0x27200a0, 0x2683540, 0xc9cfcd15f0, 0x2, 0xc9cfcd15f0, 0xc739709624)
	/usr/local/go/src/runtime/pprof/pprof.go:298 +0x341
net/http/pprof.handler.ServeHTTP(0xc739709631, 0x9, 0x2691c80, 0xc9cfcd15f0, 0xc8b54c80f0)
	/usr/local/go/src/net/http/pprof/pprof.go:209 +0x1a6
net/http/pprof.Index(0x2691c80, 0xc9cfcd15f0, 0xc8b54c80f0)
	/usr/local/go/src/net/http/pprof/pprof.go:221 +0x205
net/http.HandlerFunc.ServeHTTP(0x1be28c8, 0x2691c80, 0xc9cfcd15f0, 0xc8b54c80f0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(*ServeMux).ServeHTTP(0x2730520, 0x2691c80, 0xc9cfcd15f0, 0xc8b54c80f0)
	/usr/local/go/src/net/http/server.go:2022 +0x7f
net/http.(*ServeMux).ServeHTTP-fm(0x2691c80, 0xc9cfcd15f0, 0xc8b54c80f0)
	/go/src/github.com/prometheus/prometheus/web/web.go:214 +0x48
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x2691c80, 0xc9cfcd15f0, 0xc8b54c80f0, 0xc76404e680, 0x1, 0x1)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:84 +0x2ba
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc4202716c0, 0x2691c80, 0xc9cfcd15f0, 0xc8b54c80f0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x7ec
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0xc4205be6e0, 0x2691c80, 0xc9cfcd15f0, 0xc8b54c80f0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:126 +0x4c
net/http.serverHandler.ServeHTTP(0xc4b4186b00, 0x2691c80, 0xc9cfcd15f0, 0xc8b54c80f0)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc55feb3080, 0x2693380, 0xc60f394bc0)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 1 [select, 9850 minutes]:
main.Main(0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:228 +0x1b46
main.main()
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:43 +0x22

goroutine 120 [chan receive]:
github.com/prometheus/prometheus/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x2730ca0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/golang/glog/glog.go:879 +0x7a
created by github.com/prometheus/prometheus/vendor/github.com/golang/glog.init.1
	/go/src/github.com/prometheus/prometheus/vendor/github.com/golang/glog/glog.go:410 +0x21d

goroutine 122 [select, 7 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc4201c2300)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x5e1
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x592

goroutine 7871223 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc81f189680)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:171 +0x282
github.com/prometheus/prometheus/rules.(*Manager).ApplyConfig.func1.1(0xc420403350, 0xc81f189680)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:426 +0x52
created by github.com/prometheus/prometheus/rules.(*Manager).ApplyConfig.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:427 +0xa7

goroutine 168 [syscall, 17 minutes]:
os/signal.signal_recv(0x268abc0)
	/usr/local/go/src/runtime/sigqueue.go:116 +0x157
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 291 [select, 17 minutes]:
github.com/prometheus/prometheus/discovery.(*TargetSet).Run(0xc527c9c050, 0x7f42fe775378, 0xc65c5fc000)
	/go/src/github.com/prometheus/prometheus/discovery/discovery.go:192 +0x2e4
github.com/prometheus/prometheus/retrieval.(*TargetManager).reload.func1(0x7f42fe775378, 0xc65c5fc000, 0xc4205ed8f0, 0xc5c171e0f0)
	/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:114 +0x43
created by github.com/prometheus/prometheus/retrieval.(*TargetManager).reload
	/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:117 +0x63c

goroutine 277 [select, 9850 minutes, locked to thread]:
runtime.gopark(0x1be2d58, 0x0, 0x1aed2a1, 0x6, 0x18, 0x2)
	/usr/local/go/src/runtime/proc.go:259 +0x13a
runtime.selectgoImpl(0xc420be4f20, 0x0, 0x18)
	/usr/local/go/src/runtime/select.go:423 +0x1235
runtime.selectgo(0xc420be4f20)
	/usr/local/go/src/runtime/select.go:238 +0x1c
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal1_unix.go:304 +0x2f3
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 278 [select, 17 minutes]:
main.Main.func2(0xc420333e60, 0xc420333e00, 0xc4202f5180, 0x5, 0x5, 0xc42037c540)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:163 +0x358
created by main.Main
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:177 +0x1135

goroutine 279 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc42037ca80)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x247
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x1bc

goroutine 280 [select, 97 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc420cdc000)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x5e1
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x592

goroutine 281 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc420cdc000)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x249
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x5b4

goroutine 282 [select, 97 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc420cdc000)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x83c
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x73a

goroutine 283 [select, 105 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc420cdc000)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x263
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x75c

goroutine 284 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc4204041c0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x247
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x1bc

goroutine 123 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc4201c2300)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x249
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x5b4

goroutine 124 [select, 7 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc4201c2300)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x83c
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x73a

goroutine 125 [select, 192 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc4201c2300)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x263
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x75c

goroutine 126 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc420404460)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x247
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x1bc

goroutine 127 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc4201c2480)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x5e1
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x592

goroutine 128 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc4201c2480)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x249
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x5b4

goroutine 321 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc4201c2480)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x83c
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x73a

goroutine 322 [select, 9903 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc4201c2480)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x263
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x75c

goroutine 323 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc420404620)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x247
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x1bc

goroutine 324 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc4201c2600)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x5e1
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x592

goroutine 325 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc4201c2600)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x249
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x5b4

goroutine 326 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc4201c2600)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x83c
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x73a

goroutine 327 [select, 9903 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc4201c2600)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x263
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x75c

goroutine 328 [runnable]:
github.com/prometheus/prometheus/storage/local/codable.(*FingerprintSet).UnmarshalBinary(0xc46ba6d3d0, 0xc796e84000, 0x269508c, 0x2696000, 0xc42171c370, 0xc796e84000)
	/go/src/github.com/prometheus/prometheus/storage/local/codable/codable.go:248 +0x19e
github.com/prometheus/prometheus/storage/local/index.(*LevelDB).Get(0xc42153c6c0, 0x2684400, 0xc73a682500, 0x267da80, 0xc46ba6d3d0, 0xc95cfe5de8, 0x8cdb522bf9f415de, 0xe7f1b676ad59285e)
	/go/src/github.com/prometheus/prometheus/storage/local/index/leveldb.go:98 +0x105
github.com/prometheus/prometheus/storage/local/index.(*LabelPairFingerprintIndex).LookupSet(0xc42171c380, 0xc61f7bb3f9, 0x7, 0xc61f7bb430, 0x7, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/local/index/index.go:238 +0x101
github.com/prometheus/prometheus/storage/local.(*persistence).processIndexingQueue(0xc4201091e0)
	/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:1416 +0x96a
github.com/prometheus/prometheus/storage/local.(*persistence).run(0xc4201091e0)
	/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:344 +0x2b
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:363 +0x12f

goroutine 330 [select, 9850 minutes]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).handleQuarantine(0xc4203def20)
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1692 +0x45f
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:387 +0x3ff

goroutine 7870729 [select, 1 minutes]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc964536ee0, 0xdf8475800, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:424 +0x356
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).reload.func1
	/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:189 +0x84

goroutine 329 [select]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).handleEvictList(0xc4203def20)
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1051 +0x465
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:386 +0x3da

goroutine 375 [select, 17 minutes]:
github.com/prometheus/prometheus/discovery.(*TargetSet).Run(0xc4fa8b6b90, 0x7f42fe775378, 0xc65c6e4100)
	/go/src/github.com/prometheus/prometheus/discovery/discovery.go:192 +0x2e4
github.com/prometheus/prometheus/retrieval.(*TargetManager).reload.func1(0x7f42fe775378, 0xc65c6e4100, 0xc4205ed8f0, 0xc5c4286a80)
	/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:114 +0x43
created by github.com/prometheus/prometheus/retrieval.(*TargetManager).reload
	/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:117 +0x63c

goroutine 331 [select, 9850 minutes]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).logThrottling(0xc4203def20)
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:919 +0x838
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:388 +0x424

goroutine 332 [select]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).loop(0xc4203def20)
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1298 +0x3ca
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:389 +0x449

goroutine 433 [select, 30 minutes]:
github.com/prometheus/prometheus/notifier.(*Notifier).Run(0xc4203b6300)
	/go/src/github.com/prometheus/prometheus/notifier/notifier.go:205 +0x1b8
created by main.Main
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:208 +0x14fd

goroutine 417 [select]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).waitForNextFP(0xc4203def20, 0x279931, 0x3feed1d900000000, 0x1)
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1155 +0x198
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughMemoryFingerprints.func1(0xc5137da4e0, 0xc4203def20)
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1195 +0x245
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughMemoryFingerprints
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1205 +0x67

goroutine 418 [select]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).waitForNextFP(0xc4203def20, 0x38298, 0x3ff0000000000000, 0x38201)
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1155 +0x198
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughArchivedFingerprints.func1(0xc5137da540, 0xc4203def20)
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1239 +0x33a
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughArchivedFingerprints
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1248 +0x67

goroutine 419 [runnable]:
encoding/binary.PutVarint(0xc98a63de00, 0xa, 0x7c, 0x9, 0x7c)
	/usr/local/go/src/encoding/binary/varint.go:78
github.com/prometheus/prometheus/storage/local/codable.EncodeVarint(0x267ce80, 0xc5e7726150, 0x9, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/local/codable/codable.go:80 +0xcb
github.com/prometheus/prometheus/storage/local/codable.encodeString(0xc5e7726150, 0xc561649740, 0x9, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/local/codable/codable.go:126 +0x45
github.com/prometheus/prometheus/storage/local/codable.Metric.MarshalBinary(0xc7155a5f50, 0xc584416300, 0x3848bfdaf44a6a9c, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/local/codable/codable.go:165 +0x169
github.com/prometheus/prometheus/storage/local.(*persistence).checkpointSeriesMapAndHeads.func3(0xc420638a80, 0xc420bfee38, 0xc420bfede0, 0xc420bfeea0, 0xc584416300, 0xc4201091e0)
	/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:714 +0x1d4
github.com/prometheus/prometheus/storage/local.(*persistence).checkpointSeriesMapAndHeads(0xc4201091e0, 0xc42153ca00, 0xc420638a80, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:775 +0x4ac
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).loop.func2(0xc4203def20, 0xc5137da5a0, 0xc49d0074c0, 0xc4d2ffa4b0)
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1280 +0x162
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).loop
	/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1294 +0x149

goroutine 435 [semacquire, 9850 minutes]:
sync.runtime_Semacquire(0xc4205ed954)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc4205ed948)
	/usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/prometheus/prometheus/retrieval.(*TargetManager).Run(0xc4205ed8f0)
	/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:70 +0x15e
created by main.Main
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:214 +0x15a1

goroutine 436 [IO wait]:
net.runtime_pollWait(0x7f42f4da5f00, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4c50ce920, 0x72, 0xc58f035ce8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4c50ce920, 0x2686580, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc4c50ce8c0, 0x0, 0x2683840, 0xc76404e500)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc5818ea000, 0xc58f035dc0, 0x40a89d, 0x17b3620)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc5818ea000, 0xc5dbb14120, 0xc58f035df8, 0x43aacb, 0xc58f035e00)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
github.com/prometheus/prometheus/vendor/golang.org/x/net/netutil.(*limitListener).Accept(0xc4ecf35940, 0x1be26e8, 0xc55feb3080, 0x2693440, 0xc504cf6450)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/netutil/listen.go:30 +0x5f
net/http.(*Server).Serve(0xc4b4186b00, 0x268c8c0, 0xc4ecf35940, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
github.com/prometheus/prometheus/web.(*Handler).Run(0xc42037c540)
	/go/src/github.com/prometheus/prometheus/web/web.go:271 +0x309
created by main.Main
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:221 +0x1614

goroutine 7871298 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc94fbc8150, 0x37e11d600, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:424 +0x356
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).reload.func1
	/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:189 +0x84

goroutine 5535874 [chan send, 2797 minutes]:
github.com/prometheus/prometheus/promql.lexKeywordOrIdentifier(0xc4cc4f2360, 0x1bdbbe8)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:854 +0x3c6
github.com/prometheus/prometheus/promql.(*lexer).run(0xc4cc4f2360)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:442 +0x4c
created by github.com/prometheus/prometheus/promql.lex
	/go/src/github.com/prometheus/prometheus/promql/lex.go:435 +0xd2

goroutine 7008580 [chan send, 1041 minutes]:
github.com/prometheus/prometheus/promql.lexStatements(0xc51f6c0fc0, 0x1bdbc10)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:526 +0xa6a
github.com/prometheus/prometheus/promql.(*lexer).run(0xc51f6c0fc0)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:442 +0x4c
created by github.com/prometheus/prometheus/promql.lex
	/go/src/github.com/prometheus/prometheus/promql/lex.go:435 +0xd2

goroutine 5536309 [chan send, 2796 minutes]:
github.com/prometheus/prometheus/promql.lexKeywordOrIdentifier(0xc6c1eabec0, 0x1bdbbe8)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:854 +0x3c6
github.com/prometheus/prometheus/promql.(*lexer).run(0xc6c1eabec0)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:442 +0x4c
created by github.com/prometheus/prometheus/promql.lex
	/go/src/github.com/prometheus/prometheus/promql/lex.go:435 +0xd2

goroutine 7008226 [chan send, 1041 minutes]:
github.com/prometheus/prometheus/promql.lexKeywordOrIdentifier(0xc743090240, 0x1bdbbe8)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:854 +0x3c6
github.com/prometheus/prometheus/promql.(*lexer).run(0xc743090240)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:442 +0x4c
created by github.com/prometheus/prometheus/promql.lex
	/go/src/github.com/prometheus/prometheus/promql/lex.go:435 +0xd2

goroutine 7008213 [chan send, 1041 minutes]:
github.com/prometheus/prometheus/promql.lexStatements(0xc69ab61140, 0x1bdbc10)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:526 +0xa6a
github.com/prometheus/prometheus/promql.(*lexer).run(0xc69ab61140)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:442 +0x4c
created by github.com/prometheus/prometheus/promql.lex
	/go/src/github.com/prometheus/prometheus/promql/lex.go:435 +0xd2

goroutine 7008029 [chan send, 1042 minutes]:
github.com/prometheus/prometheus/promql.lexStatements(0xc74c36ecc0, 0x1bdbc10)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:526 +0xa6a
github.com/prometheus/prometheus/promql.(*lexer).run(0xc74c36ecc0)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:442 +0x4c
created by github.com/prometheus/prometheus/promql.lex
	/go/src/github.com/prometheus/prometheus/promql/lex.go:435 +0xd2

goroutine 7871224 [select, 17 minutes]:
github.com/prometheus/prometheus/discovery.(*TargetSet).Run(0xc8bb76caa0, 0x7f42fe775378, 0xc7881c4a00)
	/go/src/github.com/prometheus/prometheus/discovery/discovery.go:192 +0x2e4
created by github.com/prometheus/prometheus/notifier.(*Notifier).ApplyConfig
	/go/src/github.com/prometheus/prometheus/notifier/notifier.go:163 +0x384

goroutine 7007720 [chan send, 1042 minutes]:
github.com/prometheus/prometheus/promql.lexStatements(0xc6caa99b60, 0x1bdbc10)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:526 +0xa6a
github.com/prometheus/prometheus/promql.(*lexer).run(0xc6caa99b60)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:442 +0x4c
created by github.com/prometheus/prometheus/promql.lex
	/go/src/github.com/prometheus/prometheus/promql/lex.go:435 +0xd2

goroutine 5561405 [chan send, 2769 minutes]:
github.com/prometheus/prometheus/promql.lexString(0xc66d5813e0, 0x1bdbc18)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:722 +0xef
github.com/prometheus/prometheus/promql.(*lexer).run(0xc66d5813e0)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:442 +0x4c
created by github.com/prometheus/prometheus/promql.lex
	/go/src/github.com/prometheus/prometheus/promql/lex.go:435 +0xd2

goroutine 7790087 [chan send]:
github.com/prometheus/prometheus/storage/local.(*seriesMap).fpIter.func1(0xc42153ca00, 0xc825c9d4a0)
	/go/src/github.com/prometheus/prometheus/storage/local/series.go:126 +0xe0
created by github.com/prometheus/prometheus/storage/local.(*seriesMap).fpIter
	/go/src/github.com/prometheus/prometheus/storage/local/series.go:131 +0x67

goroutine 7893972 [chan send]:
github.com/prometheus/prometheus/storage/local.(*seriesMap).iter.func1(0xc42153ca00, 0xc63f91e2a0)
	/go/src/github.com/prometheus/prometheus/storage/local/series.go:104 +0x104
created by github.com/prometheus/prometheus/storage/local.(*seriesMap).iter
	/go/src/github.com/prometheus/prometheus/storage/local/series.go:109 +0x67

goroutine 7146628 [chan send, 911 minutes]:
github.com/prometheus/prometheus/promql.lexInsideBraces(0xc74fae2540, 0x1bdbbe0)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:602 +0x2be
github.com/prometheus/prometheus/promql.(*lexer).run(0xc74fae2540)
	/go/src/github.com/prometheus/prometheus/promql/lex.go:442 +0x4c
created by github.com/prometheus/prometheus/promql.lex
	/go/src/github.com/prometheus/prometheus/promql/lex.go:435 +0xd2

metrics

# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 0.039277283
go_gc_duration_seconds{quantile="0.25"} 0.045544308000000006
go_gc_duration_seconds{quantile="0.5"} 0.051463685
go_gc_duration_seconds{quantile="0.75"} 0.05962292
go_gc_duration_seconds{quantile="1"} 0.347227014
go_gc_duration_seconds_sum 1378.268359896
go_gc_duration_seconds_count 30719
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 70
# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
# TYPE go_memstats_alloc_bytes gauge
go_memstats_alloc_bytes 2.0855901384e+10
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 2.2837421492596e+14
# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
# TYPE go_memstats_buck_hash_sys_bytes gauge
go_memstats_buck_hash_sys_bytes 8.1110866e+07
# HELP go_memstats_frees_total Total number of frees.
# TYPE go_memstats_frees_total counter
go_memstats_frees_total 4.97621040739e+11
# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
# TYPE go_memstats_gc_sys_bytes gauge
go_memstats_gc_sys_bytes 1.005729792e+09
# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
# TYPE go_memstats_heap_alloc_bytes gauge
go_memstats_heap_alloc_bytes 2.0855901384e+10
# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
# TYPE go_memstats_heap_idle_bytes gauge
go_memstats_heap_idle_bytes 6.991126528e+09
# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
# TYPE go_memstats_heap_inuse_bytes gauge
go_memstats_heap_inuse_bytes 2.216718336e+10
# HELP go_memstats_heap_objects Number of allocated objects.
# TYPE go_memstats_heap_objects gauge
go_memstats_heap_objects 9.3678857e+07
# HELP go_memstats_heap_released_bytes_total Total number of heap bytes released to OS.
# TYPE go_memstats_heap_released_bytes_total counter
go_memstats_heap_released_bytes_total 1.913094144e+09
# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
# TYPE go_memstats_heap_sys_bytes gauge
go_memstats_heap_sys_bytes 2.9158309888e+10
# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
# TYPE go_memstats_last_gc_time_seconds gauge
go_memstats_last_gc_time_seconds 1.493885974039304e+09
# HELP go_memstats_lookups_total Total number of pointer lookups.
# TYPE go_memstats_lookups_total counter
go_memstats_lookups_total 5.2498644e+07
# HELP go_memstats_mallocs_total Total number of mallocs.
# TYPE go_memstats_mallocs_total counter
go_memstats_mallocs_total 4.97714719596e+11
# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
# TYPE go_memstats_mcache_inuse_bytes gauge
go_memstats_mcache_inuse_bytes 96000
# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
# TYPE go_memstats_mcache_sys_bytes gauge
go_memstats_mcache_sys_bytes 114688
# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
# TYPE go_memstats_mspan_inuse_bytes gauge
go_memstats_mspan_inuse_bytes 2.3618624e+08
# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
# TYPE go_memstats_mspan_sys_bytes gauge
go_memstats_mspan_sys_bytes 2.68304384e+08
# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
# TYPE go_memstats_next_gc_bytes gauge
go_memstats_next_gc_bytes 1.9264242145e+10
# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
# TYPE go_memstats_other_sys_bytes gauge
go_memstats_other_sys_bytes 6.5583022e+07
# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
# TYPE go_memstats_stack_inuse_bytes gauge
go_memstats_stack_inuse_bytes 8.68352e+06
# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
# TYPE go_memstats_stack_sys_bytes gauge
go_memstats_stack_sys_bytes 8.68352e+06
# HELP go_memstats_sys_bytes Number of bytes obtained by system. Sum of all system allocations.
# TYPE go_memstats_sys_bytes gauge
go_memstats_sys_bytes 3.058783616e+10
# HELP http_request_duration_microseconds The HTTP request latencies in microseconds.
# TYPE http_request_duration_microseconds summary
http_request_duration_microseconds{handler="alertmanagers",quantile="0.5"} NaN
http_request_duration_microseconds{handler="alertmanagers",quantile="0.9"} NaN
http_request_duration_microseconds{handler="alertmanagers",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="alertmanagers"} 0
http_request_duration_microseconds_count{handler="alertmanagers"} 0
http_request_duration_microseconds{handler="alerts",quantile="0.5"} NaN
http_request_duration_microseconds{handler="alerts",quantile="0.9"} NaN
http_request_duration_microseconds{handler="alerts",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="alerts"} 44653.764
http_request_duration_microseconds_count{handler="alerts"} 4
http_request_duration_microseconds{handler="config",quantile="0.5"} NaN
http_request_duration_microseconds{handler="config",quantile="0.9"} NaN
http_request_duration_microseconds{handler="config",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="config"} 3147.275
http_request_duration_microseconds_count{handler="config"} 2
http_request_duration_microseconds{handler="consoles",quantile="0.5"} NaN
http_request_duration_microseconds{handler="consoles",quantile="0.9"} NaN
http_request_duration_microseconds{handler="consoles",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="consoles"} 0
http_request_duration_microseconds_count{handler="consoles"} 0
http_request_duration_microseconds{handler="drop_series",quantile="0.5"} NaN
http_request_duration_microseconds{handler="drop_series",quantile="0.9"} NaN
http_request_duration_microseconds{handler="drop_series",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="drop_series"} 0
http_request_duration_microseconds_count{handler="drop_series"} 0
http_request_duration_microseconds{handler="federate",quantile="0.5"} 785081.804
http_request_duration_microseconds{handler="federate",quantile="0.9"} 983837.368
http_request_duration_microseconds{handler="federate",quantile="0.99"} 1.003798529e+06
http_request_duration_microseconds_sum{handler="federate"} 3.510823134672495e+10
http_request_duration_microseconds_count{handler="federate"} 46516
http_request_duration_microseconds{handler="flags",quantile="0.5"} NaN
http_request_duration_microseconds{handler="flags",quantile="0.9"} NaN
http_request_duration_microseconds{handler="flags",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="flags"} 0
http_request_duration_microseconds_count{handler="flags"} 0
http_request_duration_microseconds{handler="graph",quantile="0.5"} 1969.465
http_request_duration_microseconds{handler="graph",quantile="0.9"} 1969.465
http_request_duration_microseconds{handler="graph",quantile="0.99"} 1969.465
http_request_duration_microseconds_sum{handler="graph"} 153527.03299999997
http_request_duration_microseconds_count{handler="graph"} 33
http_request_duration_microseconds{handler="heap",quantile="0.5"} NaN
http_request_duration_microseconds{handler="heap",quantile="0.9"} NaN
http_request_duration_microseconds{handler="heap",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="heap"} 0
http_request_duration_microseconds_count{handler="heap"} 0
http_request_duration_microseconds{handler="label_values",quantile="0.5"} 5310.969
http_request_duration_microseconds{handler="label_values",quantile="0.9"} 5310.969
http_request_duration_microseconds{handler="label_values",quantile="0.99"} 5310.969
http_request_duration_microseconds_sum{handler="label_values"} 954124.236
http_request_duration_microseconds_count{handler="label_values"} 107
http_request_duration_microseconds{handler="options",quantile="0.5"} NaN
http_request_duration_microseconds{handler="options",quantile="0.9"} NaN
http_request_duration_microseconds{handler="options",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="options"} 0
http_request_duration_microseconds_count{handler="options"} 0
http_request_duration_microseconds{handler="prometheus",quantile="0.5"} 98589.817
http_request_duration_microseconds{handler="prometheus",quantile="0.9"} 191716.742
http_request_duration_microseconds{handler="prometheus",quantile="0.99"} 191716.742
http_request_duration_microseconds_sum{handler="prometheus"} 1.2872061651270013e+09
http_request_duration_microseconds_count{handler="prometheus"} 9851
http_request_duration_microseconds{handler="query",quantile="0.5"} 276.433
http_request_duration_microseconds{handler="query",quantile="0.9"} 1766.365
http_request_duration_microseconds{handler="query",quantile="0.99"} 1766.365
http_request_duration_microseconds_sum{handler="query"} 1.728117609600001e+07
http_request_duration_microseconds_count{handler="query"} 2889
http_request_duration_microseconds{handler="query_range",quantile="0.5"} 9964.8
http_request_duration_microseconds{handler="query_range",quantile="0.9"} 36876.186
http_request_duration_microseconds{handler="query_range",quantile="0.99"} 129050.437
http_request_duration_microseconds_sum{handler="query_range"} 1.7024292868544405e+10
http_request_duration_microseconds_count{handler="query_range"} 730537
http_request_duration_microseconds{handler="rules",quantile="0.5"} NaN
http_request_duration_microseconds{handler="rules",quantile="0.9"} NaN
http_request_duration_microseconds{handler="rules",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="rules"} 16249.713
http_request_duration_microseconds_count{handler="rules"} 2
http_request_duration_microseconds{handler="series",quantile="0.5"} NaN
http_request_duration_microseconds{handler="series",quantile="0.9"} NaN
http_request_duration_microseconds{handler="series",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="series"} 4.208508787e+06
http_request_duration_microseconds_count{handler="series"} 133
http_request_duration_microseconds{handler="static",quantile="0.5"} 581.921
http_request_duration_microseconds{handler="static",quantile="0.9"} 581.921
http_request_duration_microseconds{handler="static",quantile="0.99"} 581.921
http_request_duration_microseconds_sum{handler="static"} 981491.5420000001
http_request_duration_microseconds_count{handler="static"} 282
http_request_duration_microseconds{handler="status",quantile="0.5"} 4855.183
http_request_duration_microseconds{handler="status",quantile="0.9"} 4855.183
http_request_duration_microseconds{handler="status",quantile="0.99"} 4855.183
http_request_duration_microseconds_sum{handler="status"} 6817.455
http_request_duration_microseconds_count{handler="status"} 2
http_request_duration_microseconds{handler="targets",quantile="0.5"} NaN
http_request_duration_microseconds{handler="targets",quantile="0.9"} NaN
http_request_duration_microseconds{handler="targets",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="targets"} 42075.13799999999
http_request_duration_microseconds_count{handler="targets"} 8
http_request_duration_microseconds{handler="version",quantile="0.5"} NaN
http_request_duration_microseconds{handler="version",quantile="0.9"} NaN
http_request_duration_microseconds{handler="version",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="version"} 0
http_request_duration_microseconds_count{handler="version"} 0
# HELP http_request_size_bytes The HTTP request sizes in bytes.
# TYPE http_request_size_bytes summary
http_request_size_bytes{handler="alertmanagers",quantile="0.5"} NaN
http_request_size_bytes{handler="alertmanagers",quantile="0.9"} NaN
http_request_size_bytes{handler="alertmanagers",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="alertmanagers"} 0
http_request_size_bytes_count{handler="alertmanagers"} 0
http_request_size_bytes{handler="alerts",quantile="0.5"} NaN
http_request_size_bytes{handler="alerts",quantile="0.9"} NaN
http_request_size_bytes{handler="alerts",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="alerts"} 2339
http_request_size_bytes_count{handler="alerts"} 4
http_request_size_bytes{handler="config",quantile="0.5"} NaN
http_request_size_bytes{handler="config",quantile="0.9"} NaN
http_request_size_bytes{handler="config",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="config"} 1170
http_request_size_bytes_count{handler="config"} 2
http_request_size_bytes{handler="consoles",quantile="0.5"} NaN
http_request_size_bytes{handler="consoles",quantile="0.9"} NaN
http_request_size_bytes{handler="consoles",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="consoles"} 0
http_request_size_bytes_count{handler="consoles"} 0
http_request_size_bytes{handler="drop_series",quantile="0.5"} NaN
http_request_size_bytes{handler="drop_series",quantile="0.9"} NaN
http_request_size_bytes{handler="drop_series",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="drop_series"} 0
http_request_size_bytes_count{handler="drop_series"} 0
http_request_size_bytes{handler="federate",quantile="0.5"} 1352
http_request_size_bytes{handler="federate",quantile="0.9"} 1352
http_request_size_bytes{handler="federate",quantile="0.99"} 1352
http_request_size_bytes_sum{handler="federate"} 5.8076618e+07
http_request_size_bytes_count{handler="federate"} 46516
http_request_size_bytes{handler="flags",quantile="0.5"} NaN
http_request_size_bytes{handler="flags",quantile="0.9"} NaN
http_request_size_bytes{handler="flags",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="flags"} 0
http_request_size_bytes_count{handler="flags"} 0
http_request_size_bytes{handler="graph",quantile="0.5"} 677
http_request_size_bytes{handler="graph",quantile="0.9"} 677
http_request_size_bytes{handler="graph",quantile="0.99"} 677
http_request_size_bytes_sum{handler="graph"} 19467
http_request_size_bytes_count{handler="graph"} 33
http_request_size_bytes{handler="heap",quantile="0.5"} NaN
http_request_size_bytes{handler="heap",quantile="0.9"} NaN
http_request_size_bytes{handler="heap",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="heap"} 0
http_request_size_bytes_count{handler="heap"} 0
http_request_size_bytes{handler="label_values",quantile="0.5"} 742
http_request_size_bytes{handler="label_values",quantile="0.9"} 742
http_request_size_bytes{handler="label_values",quantile="0.99"} 742
http_request_size_bytes_sum{handler="label_values"} 69041
http_request_size_bytes_count{handler="label_values"} 107
http_request_size_bytes{handler="options",quantile="0.5"} NaN
http_request_size_bytes{handler="options",quantile="0.9"} NaN
http_request_size_bytes{handler="options",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="options"} 0
http_request_size_bytes_count{handler="options"} 0
http_request_size_bytes{handler="prometheus",quantile="0.5"} 238
http_request_size_bytes{handler="prometheus",quantile="0.9"} 238
http_request_size_bytes{handler="prometheus",quantile="0.99"} 238
http_request_size_bytes_sum{handler="prometheus"} 2.344538e+06
http_request_size_bytes_count{handler="prometheus"} 9851
http_request_size_bytes{handler="query",quantile="0.5"} 328
http_request_size_bytes{handler="query",quantile="0.9"} 328
http_request_size_bytes{handler="query",quantile="0.99"} 328
http_request_size_bytes_sum{handler="query"} 1.079401e+06
http_request_size_bytes_count{handler="query"} 2889
http_request_size_bytes{handler="query_range",quantile="0.5"} 861
http_request_size_bytes{handler="query_range",quantile="0.9"} 1106
http_request_size_bytes{handler="query_range",quantile="0.99"} 1338
http_request_size_bytes_sum{handler="query_range"} 5.68339377e+08
http_request_size_bytes_count{handler="query_range"} 730537
http_request_size_bytes{handler="rules",quantile="0.5"} NaN
http_request_size_bytes{handler="rules",quantile="0.9"} NaN
http_request_size_bytes{handler="rules",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="rules"} 1166
http_request_size_bytes_count{handler="rules"} 2
http_request_size_bytes{handler="series",quantile="0.5"} NaN
http_request_size_bytes{handler="series",quantile="0.9"} NaN
http_request_size_bytes{handler="series",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="series"} 100533
http_request_size_bytes_count{handler="series"} 133
http_request_size_bytes{handler="static",quantile="0.5"} 711
http_request_size_bytes{handler="static",quantile="0.9"} 711
http_request_size_bytes{handler="static",quantile="0.99"} 711
http_request_size_bytes_sum{handler="static"} 166771
http_request_size_bytes_count{handler="static"} 282
http_request_size_bytes{handler="status",quantile="0.5"} 603
http_request_size_bytes{handler="status",quantile="0.9"} 603
http_request_size_bytes{handler="status",quantile="0.99"} 603
http_request_size_bytes_sum{handler="status"} 1138
http_request_size_bytes_count{handler="status"} 2
http_request_size_bytes{handler="targets",quantile="0.5"} NaN
http_request_size_bytes{handler="targets",quantile="0.9"} NaN
http_request_size_bytes{handler="targets",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="targets"} 5060
http_request_size_bytes_count{handler="targets"} 8
http_request_size_bytes{handler="version",quantile="0.5"} NaN
http_request_size_bytes{handler="version",quantile="0.9"} NaN
http_request_size_bytes{handler="version",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="version"} 0
http_request_size_bytes_count{handler="version"} 0
# HELP http_requests_total Total number of HTTP requests made.
# TYPE http_requests_total counter
http_requests_total{code="200",handler="alerts",method="get"} 4
http_requests_total{code="200",handler="config",method="get"} 2
http_requests_total{code="200",handler="federate",method="get"} 46516
http_requests_total{code="200",handler="graph",method="get"} 33
http_requests_total{code="200",handler="label_values",method="get"} 107
http_requests_total{code="200",handler="prometheus",method="get"} 9851
http_requests_total{code="200",handler="query",method="get"} 2886
http_requests_total{code="200",handler="query_range",method="get"} 730530
http_requests_total{code="200",handler="rules",method="get"} 2
http_requests_total{code="200",handler="series",method="get"} 133
http_requests_total{code="200",handler="static",method="get"} 255
http_requests_total{code="200",handler="status",method="get"} 2
http_requests_total{code="200",handler="targets",method="get"} 8
http_requests_total{code="304",handler="static",method="get"} 27
http_requests_total{code="400",handler="query",method="get"} 3
http_requests_total{code="400",handler="query_range",method="get"} 7
# HELP http_response_size_bytes The HTTP response sizes in bytes.
# TYPE http_response_size_bytes summary
http_response_size_bytes{handler="alertmanagers",quantile="0.5"} NaN
http_response_size_bytes{handler="alertmanagers",quantile="0.9"} NaN
http_response_size_bytes{handler="alertmanagers",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="alertmanagers"} 0
http_response_size_bytes_count{handler="alertmanagers"} 0
http_response_size_bytes{handler="alerts",quantile="0.5"} NaN
http_response_size_bytes{handler="alerts",quantile="0.9"} NaN
http_response_size_bytes{handler="alerts",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="alerts"} 196815
http_response_size_bytes_count{handler="alerts"} 4
http_response_size_bytes{handler="config",quantile="0.5"} NaN
http_response_size_bytes{handler="config",quantile="0.9"} NaN
http_response_size_bytes{handler="config",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="config"} 12692
http_response_size_bytes_count{handler="config"} 2
http_response_size_bytes{handler="consoles",quantile="0.5"} NaN
http_response_size_bytes{handler="consoles",quantile="0.9"} NaN
http_response_size_bytes{handler="consoles",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="consoles"} 0
http_response_size_bytes_count{handler="consoles"} 0
http_response_size_bytes{handler="drop_series",quantile="0.5"} NaN
http_response_size_bytes{handler="drop_series",quantile="0.9"} NaN
http_response_size_bytes{handler="drop_series",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="drop_series"} 0
http_response_size_bytes_count{handler="drop_series"} 0
http_response_size_bytes{handler="federate",quantile="0.5"} 330856
http_response_size_bytes{handler="federate",quantile="0.9"} 347469
http_response_size_bytes{handler="federate",quantile="0.99"} 347805
http_response_size_bytes_sum{handler="federate"} 1.2955332106e+10
http_response_size_bytes_count{handler="federate"} 46516
http_response_size_bytes{handler="flags",quantile="0.5"} NaN
http_response_size_bytes{handler="flags",quantile="0.9"} NaN
http_response_size_bytes{handler="flags",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="flags"} 0
http_response_size_bytes_count{handler="flags"} 0
http_response_size_bytes{handler="graph",quantile="0.5"} 3681
http_response_size_bytes{handler="graph",quantile="0.9"} 3681
http_response_size_bytes{handler="graph",quantile="0.99"} 3681
http_response_size_bytes_sum{handler="graph"} 121473
http_response_size_bytes_count{handler="graph"} 33
http_response_size_bytes{handler="heap",quantile="0.5"} NaN
http_response_size_bytes{handler="heap",quantile="0.9"} NaN
http_response_size_bytes{handler="heap",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="heap"} 0
http_response_size_bytes_count{handler="heap"} 0
http_response_size_bytes{handler="label_values",quantile="0.5"} 8897
http_response_size_bytes{handler="label_values",quantile="0.9"} 8897
http_response_size_bytes{handler="label_values",quantile="0.99"} 8897
http_response_size_bytes_sum{handler="label_values"} 926727
http_response_size_bytes_count{handler="label_values"} 107
http_response_size_bytes{handler="options",quantile="0.5"} NaN
http_response_size_bytes{handler="options",quantile="0.9"} NaN
http_response_size_bytes{handler="options",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="options"} 0
http_response_size_bytes_count{handler="options"} 0
http_response_size_bytes{handler="prometheus",quantile="0.5"} 6453
http_response_size_bytes{handler="prometheus",quantile="0.9"} 6460
http_response_size_bytes{handler="prometheus",quantile="0.99"} 6460
http_response_size_bytes_sum{handler="prometheus"} 5.9977409e+07
http_response_size_bytes_count{handler="prometheus"} 9851
http_response_size_bytes{handler="query",quantile="0.5"} 63
http_response_size_bytes{handler="query",quantile="0.9"} 63
http_response_size_bytes{handler="query",quantile="0.99"} 63
http_response_size_bytes_sum{handler="query"} 298761
http_response_size_bytes_count{handler="query"} 2889
http_response_size_bytes{handler="query_range",quantile="0.5"} 745
http_response_size_bytes{handler="query_range",quantile="0.9"} 1740
http_response_size_bytes{handler="query_range",quantile="0.99"} 8190
http_response_size_bytes_sum{handler="query_range"} 5.09364578e+08
http_response_size_bytes_count{handler="query_range"} 730537
http_response_size_bytes{handler="rules",quantile="0.5"} NaN
http_response_size_bytes{handler="rules",quantile="0.9"} NaN
http_response_size_bytes{handler="rules",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="rules"} 86420
http_response_size_bytes_count{handler="rules"} 2
http_response_size_bytes{handler="series",quantile="0.5"} NaN
http_response_size_bytes{handler="series",quantile="0.9"} NaN
http_response_size_bytes{handler="series",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="series"} 67339
http_response_size_bytes_count{handler="series"} 133
http_response_size_bytes{handler="static",quantile="0.5"} 6316
http_response_size_bytes{handler="static",quantile="0.9"} 6316
http_response_size_bytes{handler="static",quantile="0.99"} 6316
http_response_size_bytes_sum{handler="static"} 7.700398e+06
http_response_size_bytes_count{handler="static"} 282
http_response_size_bytes{handler="status",quantile="0.5"} 3880
http_response_size_bytes{handler="status",quantile="0.9"} 3880
http_response_size_bytes{handler="status",quantile="0.99"} 3880
http_response_size_bytes_sum{handler="status"} 7760
http_response_size_bytes_count{handler="status"} 2
http_response_size_bytes{handler="targets",quantile="0.5"} NaN
http_response_size_bytes{handler="targets",quantile="0.9"} NaN
http_response_size_bytes{handler="targets",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="targets"} 124383
http_response_size_bytes_count{handler="targets"} 8
http_response_size_bytes{handler="version",quantile="0.5"} NaN
http_response_size_bytes{handler="version",quantile="0.9"} NaN
http_response_size_bytes{handler="version",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="version"} 0
http_response_size_bytes_count{handler="version"} 0
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 2.00030861e+06
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 1.048576e+06
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 469
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 2.8685463552e+10
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.49329170542e+09
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 3.0625046528e+10
# HELP prometheus_build_info A metric with a constant '1' value labeled by version, revision, branch, and goversion from which prometheus was built.
# TYPE prometheus_build_info gauge
prometheus_build_info{branch="master",goversion="go1.7.5",revision="bd1182d29f462c39544f94cc822830e1c64cf55b",version="1.5.2"} 1
# HELP prometheus_config_last_reload_success_timestamp_seconds Timestamp of the last successful configuration reload.
# TYPE prometheus_config_last_reload_success_timestamp_seconds gauge
prometheus_config_last_reload_success_timestamp_seconds 1.493884863e+09
# HELP prometheus_config_last_reload_successful Whether the last configuration reload attempt was successful.
# TYPE prometheus_config_last_reload_successful gauge
prometheus_config_last_reload_successful 1
# HELP prometheus_engine_queries The current number of queries being executed or waiting.
# TYPE prometheus_engine_queries gauge
prometheus_engine_queries 0
# HELP prometheus_engine_queries_concurrent_max The max number of concurrent queries.
# TYPE prometheus_engine_queries_concurrent_max gauge
prometheus_engine_queries_concurrent_max 20
# HELP prometheus_evaluator_duration_seconds The duration of rule group evaluations.
# TYPE prometheus_evaluator_duration_seconds summary
prometheus_evaluator_duration_seconds{quantile="0.01"} 1.111506038
prometheus_evaluator_duration_seconds{quantile="0.05"} 1.111506038
prometheus_evaluator_duration_seconds{quantile="0.5"} 1.735347046
prometheus_evaluator_duration_seconds{quantile="0.9"} 3.706672713
prometheus_evaluator_duration_seconds{quantile="0.99"} 3.706672713
prometheus_evaluator_duration_seconds_sum 26977.458791266934
prometheus_evaluator_duration_seconds_count 9851
# HELP prometheus_evaluator_iterations_skipped_total The total number of rule group evaluations skipped due to throttled metric storage.
# TYPE prometheus_evaluator_iterations_skipped_total counter
prometheus_evaluator_iterations_skipped_total 0
# HELP prometheus_local_storage_checkpoint_duration_seconds The duration in seconds taken for checkpointing open chunks and chunks yet to be persisted
# TYPE prometheus_local_storage_checkpoint_duration_seconds summary
prometheus_local_storage_checkpoint_duration_seconds{quantile="0.5"} 114.50714994
prometheus_local_storage_checkpoint_duration_seconds{quantile="0.9"} 114.50714994
prometheus_local_storage_checkpoint_duration_seconds{quantile="0.99"} 114.50714994
prometheus_local_storage_checkpoint_duration_seconds_sum 140459.13553946614
prometheus_local_storage_checkpoint_duration_seconds_count 1699
# HELP prometheus_local_storage_checkpoint_last_duration_seconds The duration in seconds it took to last checkpoint open chunks and chunks yet to be persisted.
# TYPE prometheus_local_storage_checkpoint_last_duration_seconds gauge
prometheus_local_storage_checkpoint_last_duration_seconds 114.50714994
# HELP prometheus_local_storage_checkpoint_last_size_bytes The size of the last checkpoint of open chunks and chunks yet to be persisted
# TYPE prometheus_local_storage_checkpoint_last_size_bytes gauge
prometheus_local_storage_checkpoint_last_size_bytes 1.622953744e+09
# HELP prometheus_local_storage_checkpoint_series_chunks_written The number of chunk written per series while checkpointing open chunks and chunks yet to be persisted.
# TYPE prometheus_local_storage_checkpoint_series_chunks_written summary
prometheus_local_storage_checkpoint_series_chunks_written{quantile="0.5"} 1
prometheus_local_storage_checkpoint_series_chunks_written{quantile="0.9"} 3
prometheus_local_storage_checkpoint_series_chunks_written{quantile="0.99"} 5
prometheus_local_storage_checkpoint_series_chunks_written_sum 1.013707347e+09
prometheus_local_storage_checkpoint_series_chunks_written_count 6.00228785e+08
# HELP prometheus_local_storage_checkpointing 1 if the storage is checkpointing, 0 otherwise.
# TYPE prometheus_local_storage_checkpointing gauge
prometheus_local_storage_checkpointing 0
# HELP prometheus_local_storage_chunk_ops_total The total number of chunk operations by their type.
# TYPE prometheus_local_storage_chunk_ops_total counter
prometheus_local_storage_chunk_ops_total{type="clone"} 6878
prometheus_local_storage_chunk_ops_total{type="create"} 8.41642e+06
prometheus_local_storage_chunk_ops_total{type="drop"} 8.507768e+06
prometheus_local_storage_chunk_ops_total{type="evict"} 4.223218e+06
prometheus_local_storage_chunk_ops_total{type="load"} 2510
prometheus_local_storage_chunk_ops_total{type="persist"} 8.062143e+06
prometheus_local_storage_chunk_ops_total{type="pin"} 5.1847944e+07
prometheus_local_storage_chunk_ops_total{type="transcode"} 5.072795e+06
prometheus_local_storage_chunk_ops_total{type="unpin"} 5.1847944e+07
# HELP prometheus_local_storage_chunkdesc_ops_total The total number of chunk descriptor operations by their type.
# TYPE prometheus_local_storage_chunkdesc_ops_total counter
prometheus_local_storage_chunkdesc_ops_total{type="evict"} 1.475145e+06
prometheus_local_storage_chunkdesc_ops_total{type="load"} 87286
# HELP prometheus_local_storage_chunks_to_persist The current number of chunks waiting for persistence.
# TYPE prometheus_local_storage_chunks_to_persist gauge
prometheus_local_storage_chunks_to_persist 78412
# HELP prometheus_local_storage_fingerprint_mappings_total The total number of fingerprints being mapped to avoid collisions.
# TYPE prometheus_local_storage_fingerprint_mappings_total counter
prometheus_local_storage_fingerprint_mappings_total 0
# HELP prometheus_local_storage_inconsistencies_total A counter incremented each time an inconsistency in the local storage is detected. If this is greater zero, restart the server as soon as possible.
# TYPE prometheus_local_storage_inconsistencies_total counter
prometheus_local_storage_inconsistencies_total 0
# HELP prometheus_local_storage_indexing_batch_duration_seconds Quantiles for batch indexing duration in seconds.
# TYPE prometheus_local_storage_indexing_batch_duration_seconds summary
prometheus_local_storage_indexing_batch_duration_seconds{quantile="0.5"} 17.635035275
prometheus_local_storage_indexing_batch_duration_seconds{quantile="0.9"} 18.767807005
prometheus_local_storage_indexing_batch_duration_seconds{quantile="0.99"} 19.370643307
prometheus_local_storage_indexing_batch_duration_seconds_sum 297634.63426821906
prometheus_local_storage_indexing_batch_duration_seconds_count 17826
# HELP prometheus_local_storage_indexing_batch_sizes Quantiles for indexing batch sizes (number of metrics per batch).
# TYPE prometheus_local_storage_indexing_batch_sizes summary
prometheus_local_storage_indexing_batch_sizes{quantile="0.5"} 257
prometheus_local_storage_indexing_batch_sizes{quantile="0.9"} 473
prometheus_local_storage_indexing_batch_sizes{quantile="0.99"} 522
prometheus_local_storage_indexing_batch_sizes_sum 1.4926645e+07
prometheus_local_storage_indexing_batch_sizes_count 17826
# HELP prometheus_local_storage_indexing_queue_capacity The capacity of the indexing queue.
# TYPE prometheus_local_storage_indexing_queue_capacity gauge
prometheus_local_storage_indexing_queue_capacity 262144
# HELP prometheus_local_storage_indexing_queue_length The number of metrics waiting to be indexed.
# TYPE prometheus_local_storage_indexing_queue_length gauge
prometheus_local_storage_indexing_queue_length 278
# HELP prometheus_local_storage_ingested_samples_total The total number of samples ingested.
# TYPE prometheus_local_storage_ingested_samples_total counter
prometheus_local_storage_ingested_samples_total 1.227627381e+09
# HELP prometheus_local_storage_maintain_series_duration_seconds The duration in seconds it took to perform maintenance on a series.
# TYPE prometheus_local_storage_maintain_series_duration_seconds summary
prometheus_local_storage_maintain_series_duration_seconds{location="archived",quantile="0.5"} 0.010478486
prometheus_local_storage_maintain_series_duration_seconds{location="archived",quantile="0.9"} 0.016538574
prometheus_local_storage_maintain_series_duration_seconds{location="archived",quantile="0.99"} 0.038089554000000005
prometheus_local_storage_maintain_series_duration_seconds_sum{location="archived"} 77982.53322825879
prometheus_local_storage_maintain_series_duration_seconds_count{location="archived"} 7.139287e+06
prometheus_local_storage_maintain_series_duration_seconds{location="memory",quantile="0.5"} 0.0005651590000000001
prometheus_local_storage_maintain_series_duration_seconds{location="memory",quantile="0.9"} 0.002469881
prometheus_local_storage_maintain_series_duration_seconds{location="memory",quantile="0.99"} 0.013098945
prometheus_local_storage_maintain_series_duration_seconds_sum{location="memory"} 77572.41122077253
prometheus_local_storage_maintain_series_duration_seconds_count{location="memory"} 5.2128507e+07
# HELP prometheus_local_storage_max_chunks_to_persist The maximum number of chunks that can be waiting for persistence before sample ingestion will stop.
# TYPE prometheus_local_storage_max_chunks_to_persist gauge
prometheus_local_storage_max_chunks_to_persist 2.097152e+06
# HELP prometheus_local_storage_max_memory_chunks The configured maximum number of chunks that can be held in memory
# TYPE prometheus_local_storage_max_memory_chunks gauge
prometheus_local_storage_max_memory_chunks 4.194304e+06
# HELP prometheus_local_storage_memory_chunkdescs The current number of chunk descriptors in memory.
# TYPE prometheus_local_storage_memory_chunkdescs gauge
prometheus_local_storage_memory_chunkdescs 7.547325e+06
# HELP prometheus_local_storage_memory_chunks The current number of chunks in memory, excluding cloned chunks (i.e. chunks without a descriptor).
# TYPE prometheus_local_storage_memory_chunks gauge
prometheus_local_storage_memory_chunks 4.195712e+06
# HELP prometheus_local_storage_memory_dirty_series The current number of series that would require a disk seek during crash recovery.
# TYPE prometheus_local_storage_memory_dirty_series gauge
prometheus_local_storage_memory_dirty_series 876
# HELP prometheus_local_storage_memory_series The current number of series in memory.
# TYPE prometheus_local_storage_memory_series gauge
prometheus_local_storage_memory_series 2.595222e+06
# HELP prometheus_local_storage_non_existent_series_matches_total How often a non-existent series was referred to during label matching or chunk preloading. This is an indication of outdated label indexes.
# TYPE prometheus_local_storage_non_existent_series_matches_total counter
prometheus_local_storage_non_existent_series_matches_total 274
# HELP prometheus_local_storage_out_of_order_samples_total The total number of samples that were discarded because their timestamps were at or before the last received sample for a series.
# TYPE prometheus_local_storage_out_of_order_samples_total counter
prometheus_local_storage_out_of_order_samples_total{reason="multiple_values_for_timestamp"} 0
prometheus_local_storage_out_of_order_samples_total{reason="timestamp_out_of_order"} 39034
# HELP prometheus_local_storage_persist_errors_total The total number of errors while persisting chunks.
# TYPE prometheus_local_storage_persist_errors_total counter
prometheus_local_storage_persist_errors_total 0
# HELP prometheus_local_storage_persistence_urgency_score A score of urgency to persist chunks, 0 is least urgent, 1 most.
# TYPE prometheus_local_storage_persistence_urgency_score gauge
prometheus_local_storage_persistence_urgency_score 0.03738975524902344
# HELP prometheus_local_storage_queued_chunks_to_persist_total The total number of chunks queued for persistence.
# TYPE prometheus_local_storage_queued_chunks_to_persist_total counter
prometheus_local_storage_queued_chunks_to_persist_total 8.092046e+06
# HELP prometheus_local_storage_rushed_mode 1 if the storage is in rushed mode, 0 otherwise. In rushed mode, the system behaves as if the persistence_urgency_score is 1.
# TYPE prometheus_local_storage_rushed_mode gauge
prometheus_local_storage_rushed_mode 0
# HELP prometheus_local_storage_series_chunks_persisted The number of chunks persisted per series.
# TYPE prometheus_local_storage_series_chunks_persisted histogram
prometheus_local_storage_series_chunks_persisted_bucket{le="1"} 5.780257e+06
prometheus_local_storage_series_chunks_persisted_bucket{le="2"} 6.221595e+06
prometheus_local_storage_series_chunks_persisted_bucket{le="4"} 6.473165e+06
prometheus_local_storage_series_chunks_persisted_bucket{le="8"} 6.570852e+06
prometheus_local_storage_series_chunks_persisted_bucket{le="16"} 6.571118e+06
prometheus_local_storage_series_chunks_persisted_bucket{le="32"} 6.571127e+06
prometheus_local_storage_series_chunks_persisted_bucket{le="64"} 6.571127e+06
prometheus_local_storage_series_chunks_persisted_bucket{le="128"} 6.571127e+06
prometheus_local_storage_series_chunks_persisted_bucket{le="+Inf"} 6.571127e+06
prometheus_local_storage_series_chunks_persisted_sum 8.062143e+06
prometheus_local_storage_series_chunks_persisted_count 6.571127e+06
# HELP prometheus_local_storage_series_ops_total The total number of series operations by their type.
# TYPE prometheus_local_storage_series_ops_total counter
prometheus_local_storage_series_ops_total{type="archive"} 3.026537e+06
prometheus_local_storage_series_ops_total{type="create"} 4.812928e+06
prometheus_local_storage_series_ops_total{type="maintenance_in_archive"} 7.137181e+06
prometheus_local_storage_series_ops_total{type="maintenance_in_memory"} 5.2128507e+07
prometheus_local_storage_series_ops_total{type="purge_from_archive"} 4.860374e+06
prometheus_local_storage_series_ops_total{type="purge_from_memory"} 34
prometheus_local_storage_series_ops_total{type="purge_on_request"} 0
prometheus_local_storage_series_ops_total{type="quarantine_completed"} 0
prometheus_local_storage_series_ops_total{type="quarantine_dropped"} 0
prometheus_local_storage_series_ops_total{type="quarantine_failed"} 0
prometheus_local_storage_series_ops_total{type="unarchive"} 8738
# HELP prometheus_local_storage_started_dirty Whether the local storage was found to be dirty (and crash recovery occurred) during Prometheus startup.
# TYPE prometheus_local_storage_started_dirty gauge
prometheus_local_storage_started_dirty 1
# HELP prometheus_notifications_dropped_total Total number of alerts dropped due to errors when sending to Alertmanager.
# TYPE prometheus_notifications_dropped_total counter
prometheus_notifications_dropped_total 75
# HELP prometheus_notifications_errors_total Total number of errors sending alert notifications.
# TYPE prometheus_notifications_errors_total counter
prometheus_notifications_errors_total{alertmanager="https://alertmanager.global.cloud.sap:443/api/v1/alerts"} 44
# HELP prometheus_notifications_latency_seconds Latency quantiles for sending alert notifications (not including dropped notifications).
# TYPE prometheus_notifications_latency_seconds summary
prometheus_notifications_latency_seconds{alertmanager="https://alertmanager.global.cloud.sap:443/api/v1/alerts",quantile="0.5"} NaN
prometheus_notifications_latency_seconds{alertmanager="https://alertmanager.global.cloud.sap:443/api/v1/alerts",quantile="0.9"} NaN
prometheus_notifications_latency_seconds{alertmanager="https://alertmanager.global.cloud.sap:443/api/v1/alerts",quantile="0.99"} NaN
prometheus_notifications_latency_seconds_sum{alertmanager="https://alertmanager.global.cloud.sap:443/api/v1/alerts"} 533.9525254309947
prometheus_notifications_latency_seconds_count{alertmanager="https://alertmanager.global.cloud.sap:443/api/v1/alerts"} 23512
# HELP prometheus_notifications_queue_capacity The capacity of the alert notifications queue.
# TYPE prometheus_notifications_queue_capacity gauge
prometheus_notifications_queue_capacity 10000
# HELP prometheus_notifications_queue_length The number of alert notifications in the queue.
# TYPE prometheus_notifications_queue_length gauge
prometheus_notifications_queue_length 0
# HELP prometheus_notifications_sent_total Total number of alerts successfully sent.
# TYPE prometheus_notifications_sent_total counter
prometheus_notifications_sent_total{alertmanager="https://alertmanager.global.cloud.sap:443/api/v1/alerts"} 35935
# HELP prometheus_rule_evaluation_duration_seconds The duration for a rule to execute.
# TYPE prometheus_rule_evaluation_duration_seconds summary
prometheus_rule_evaluation_duration_seconds{rule_type="alerting",quantile="0.5"} 0.013919722
prometheus_rule_evaluation_duration_seconds{rule_type="alerting",quantile="0.9"} 0.301395387
prometheus_rule_evaluation_duration_seconds{rule_type="alerting",quantile="0.99"} 0.32783395200000004
prometheus_rule_evaluation_duration_seconds_sum{rule_type="alerting"} 9578.320498448884
prometheus_rule_evaluation_duration_seconds_count{rule_type="alerting"} 413742
prometheus_rule_evaluation_duration_seconds{rule_type="recording",quantile="0.5"} 0.055696419000000004
prometheus_rule_evaluation_duration_seconds{rule_type="recording",quantile="0.9"} 1.511686499
prometheus_rule_evaluation_duration_seconds{rule_type="recording",quantile="0.99"} 3.70623739
prometheus_rule_evaluation_duration_seconds_sum{rule_type="recording"} 149308.61397526154
prometheus_rule_evaluation_duration_seconds_count{rule_type="recording"} 265977
# HELP prometheus_rule_evaluation_failures_total The total number of rule evaluation failures.
# TYPE prometheus_rule_evaluation_failures_total counter
prometheus_rule_evaluation_failures_total{rule_type="alerting"} 0
prometheus_rule_evaluation_failures_total{rule_type="recording"} 0
# HELP prometheus_sd_azure_refresh_duration_seconds The duration of a Azure-SD refresh in seconds.
# TYPE prometheus_sd_azure_refresh_duration_seconds summary
prometheus_sd_azure_refresh_duration_seconds{quantile="0.5"} NaN
prometheus_sd_azure_refresh_duration_seconds{quantile="0.9"} NaN
prometheus_sd_azure_refresh_duration_seconds{quantile="0.99"} NaN
prometheus_sd_azure_refresh_duration_seconds_sum 0
prometheus_sd_azure_refresh_duration_seconds_count 0
# HELP prometheus_sd_azure_refresh_failures_total Number of Azure-SD refresh failures.
# TYPE prometheus_sd_azure_refresh_failures_total counter
prometheus_sd_azure_refresh_failures_total 0
# HELP prometheus_sd_consul_rpc_duration_seconds The duration of a Consul RPC call in seconds.
# TYPE prometheus_sd_consul_rpc_duration_seconds summary
prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.5"} NaN
prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.9"} NaN
prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.99"} NaN
prometheus_sd_consul_rpc_duration_seconds_sum{call="service",endpoint="catalog"} 0
prometheus_sd_consul_rpc_duration_seconds_count{call="service",endpoint="catalog"} 0
prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.5"} NaN
prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.9"} NaN
prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.99"} NaN
prometheus_sd_consul_rpc_duration_seconds_sum{call="services",endpoint="catalog"} 0
prometheus_sd_consul_rpc_duration_seconds_count{call="services",endpoint="catalog"} 0
# HELP prometheus_sd_consul_rpc_failures_total The number of Consul RPC call failures.
# TYPE prometheus_sd_consul_rpc_failures_total counter
prometheus_sd_consul_rpc_failures_total 0
# HELP prometheus_sd_dns_lookup_failures_total The number of DNS-SD lookup failures.
# TYPE prometheus_sd_dns_lookup_failures_total counter
prometheus_sd_dns_lookup_failures_total 0
# HELP prometheus_sd_dns_lookups_total The number of DNS-SD lookups.
# TYPE prometheus_sd_dns_lookups_total counter
prometheus_sd_dns_lookups_total 0
# HELP prometheus_sd_ec2_refresh_duration_seconds The duration of a EC2-SD refresh in seconds.
# TYPE prometheus_sd_ec2_refresh_duration_seconds summary
prometheus_sd_ec2_refresh_duration_seconds{quantile="0.5"} NaN
prometheus_sd_ec2_refresh_duration_seconds{quantile="0.9"} NaN
prometheus_sd_ec2_refresh_duration_seconds{quantile="0.99"} NaN
prometheus_sd_ec2_refresh_duration_seconds_sum 0
prometheus_sd_ec2_refresh_duration_seconds_count 0
# HELP prometheus_sd_ec2_refresh_failures_total The number of EC2-SD scrape failures.
# TYPE prometheus_sd_ec2_refresh_failures_total counter
prometheus_sd_ec2_refresh_failures_total 0
# HELP prometheus_sd_file_read_errors_total The number of File-SD read errors.
# TYPE prometheus_sd_file_read_errors_total counter
prometheus_sd_file_read_errors_total 0
# HELP prometheus_sd_file_scan_duration_seconds The duration of the File-SD scan in seconds.
# TYPE prometheus_sd_file_scan_duration_seconds summary
prometheus_sd_file_scan_duration_seconds{quantile="0.5"} NaN
prometheus_sd_file_scan_duration_seconds{quantile="0.9"} NaN
prometheus_sd_file_scan_duration_seconds{quantile="0.99"} NaN
prometheus_sd_file_scan_duration_seconds_sum 0
prometheus_sd_file_scan_duration_seconds_count 0
# HELP prometheus_sd_gce_refresh_duration The duration of a GCE-SD refresh in seconds.
# TYPE prometheus_sd_gce_refresh_duration summary
prometheus_sd_gce_refresh_duration{quantile="0.5"} NaN
prometheus_sd_gce_refresh_duration{quantile="0.9"} NaN
prometheus_sd_gce_refresh_duration{quantile="0.99"} NaN
prometheus_sd_gce_refresh_duration_sum 0
prometheus_sd_gce_refresh_duration_count 0
# HELP prometheus_sd_gce_refresh_failures_total The number of GCE-SD refresh failures.
# TYPE prometheus_sd_gce_refresh_failures_total counter
prometheus_sd_gce_refresh_failures_total 0
# HELP prometheus_sd_kubernetes_events_total The number of Kubernetes events handled.
# TYPE prometheus_sd_kubernetes_events_total counter
prometheus_sd_kubernetes_events_total{event="add",role="endpoints"} 0
prometheus_sd_kubernetes_events_total{event="add",role="node"} 0
prometheus_sd_kubernetes_events_total{event="add",role="pod"} 0
prometheus_sd_kubernetes_events_total{event="add",role="service"} 0
prometheus_sd_kubernetes_events_total{event="delete",role="endpoints"} 0
prometheus_sd_kubernetes_events_total{event="delete",role="node"} 0
prometheus_sd_kubernetes_events_total{event="delete",role="pod"} 0
prometheus_sd_kubernetes_events_total{event="delete",role="service"} 0
prometheus_sd_kubernetes_events_total{event="update",role="endpoints"} 0
prometheus_sd_kubernetes_events_total{event="update",role="node"} 0
prometheus_sd_kubernetes_events_total{event="update",role="pod"} 0
prometheus_sd_kubernetes_events_total{event="update",role="service"} 0
# HELP prometheus_sd_marathon_refresh_duration_seconds The duration of a Marathon-SD refresh in seconds.
# TYPE prometheus_sd_marathon_refresh_duration_seconds summary
prometheus_sd_marathon_refresh_duration_seconds{quantile="0.5"} NaN
prometheus_sd_marathon_refresh_duration_seconds{quantile="0.9"} NaN
prometheus_sd_marathon_refresh_duration_seconds{quantile="0.99"} NaN
prometheus_sd_marathon_refresh_duration_seconds_sum 0
prometheus_sd_marathon_refresh_duration_seconds_count 0
# HELP prometheus_sd_marathon_refresh_failures_total The number of Marathon-SD refresh failures.
# TYPE prometheus_sd_marathon_refresh_failures_total counter
prometheus_sd_marathon_refresh_failures_total 0
# HELP prometheus_sd_triton_refresh_duration_seconds The duration of a Triton-SD refresh in seconds.
# TYPE prometheus_sd_triton_refresh_duration_seconds summary
prometheus_sd_triton_refresh_duration_seconds{quantile="0.5"} NaN
prometheus_sd_triton_refresh_duration_seconds{quantile="0.9"} NaN
prometheus_sd_triton_refresh_duration_seconds{quantile="0.99"} NaN
prometheus_sd_triton_refresh_duration_seconds_sum 0
prometheus_sd_triton_refresh_duration_seconds_count 0
# HELP prometheus_sd_triton_refresh_failures_total The number of Triton-SD scrape failures.
# TYPE prometheus_sd_triton_refresh_failures_total counter
prometheus_sd_triton_refresh_failures_total 0
# HELP prometheus_target_interval_length_seconds Actual intervals between scrapes.
# TYPE prometheus_target_interval_length_seconds summary
prometheus_target_interval_length_seconds{interval="15s",quantile="0.01"} 14.999507512000001
prometheus_target_interval_length_seconds{interval="15s",quantile="0.05"} 14.999507512000001
prometheus_target_interval_length_seconds{interval="15s",quantile="0.5"} 15.00002871
prometheus_target_interval_length_seconds{interval="15s",quantile="0.9"} 15.000201985
prometheus_target_interval_length_seconds{interval="15s",quantile="0.99"} 15.000241532
prometheus_target_interval_length_seconds_sum{interval="15s"} 590948.7147672203
prometheus_target_interval_length_seconds_count{interval="15s"} 39396
prometheus_target_interval_length_seconds{interval="1m0s",quantile="0.01"} 59.994038835
prometheus_target_interval_length_seconds{interval="1m0s",quantile="0.05"} 59.994038835
prometheus_target_interval_length_seconds{interval="1m0s",quantile="0.5"} 60.000027396
prometheus_target_interval_length_seconds{interval="1m0s",quantile="0.9"} 60.0036935
prometheus_target_interval_length_seconds{interval="1m0s",quantile="0.99"} 60.0036935
prometheus_target_interval_length_seconds_sum{interval="1m0s"} 590702.2994885265
prometheus_target_interval_length_seconds_count{interval="1m0s"} 9845
# HELP prometheus_target_reload_length_seconds Actual interval to reload the scrape pool with a given configuration.
# TYPE prometheus_target_reload_length_seconds summary
prometheus_target_reload_length_seconds{interval="15s",quantile="0.01"} NaN
prometheus_target_reload_length_seconds{interval="15s",quantile="0.05"} NaN
prometheus_target_reload_length_seconds{interval="15s",quantile="0.5"} NaN
prometheus_target_reload_length_seconds{interval="15s",quantile="0.9"} NaN
prometheus_target_reload_length_seconds{interval="15s",quantile="0.99"} NaN
prometheus_target_reload_length_seconds_sum{interval="15s"} 0.0029323030000000003
prometheus_target_reload_length_seconds_count{interval="15s"} 8
prometheus_target_reload_length_seconds{interval="1m0s",quantile="0.01"} NaN
prometheus_target_reload_length_seconds{interval="1m0s",quantile="0.05"} NaN
prometheus_target_reload_length_seconds{interval="1m0s",quantile="0.5"} NaN
prometheus_target_reload_length_seconds{interval="1m0s",quantile="0.9"} NaN
prometheus_target_reload_length_seconds{interval="1m0s",quantile="0.99"} NaN
prometheus_target_reload_length_seconds_sum{interval="1m0s"} 0.001192687
prometheus_target_reload_length_seconds_count{interval="1m0s"} 8
# HELP prometheus_target_scrape_pool_sync_total Total number of syncs that were executed on a scrape pool.
# TYPE prometheus_target_scrape_pool_sync_total counter
prometheus_target_scrape_pool_sync_total{scrape_job="prometheus-collector-federation"} 9
prometheus_target_scrape_pool_sync_total{scrape_job="prometheus-frontend"} 9
# HELP prometheus_target_scrapes_exceeded_sample_limit_total Total number of scrapes that hit the sample limit and were rejected.
# TYPE prometheus_target_scrapes_exceeded_sample_limit_total counter
prometheus_target_scrapes_exceeded_sample_limit_total 0
# HELP prometheus_target_skipped_scrapes_total Total number of scrapes that were skipped because the metric storage was throttled.
# TYPE prometheus_target_skipped_scrapes_total counter
prometheus_target_skipped_scrapes_total 0
# HELP prometheus_target_sync_length_seconds Actual interval to sync the scrape pool.
# TYPE prometheus_target_sync_length_seconds summary
prometheus_target_sync_length_seconds{scrape_job="prometheus-collector-federation",quantile="0.01"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus-collector-federation",quantile="0.05"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus-collector-federation",quantile="0.5"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus-collector-federation",quantile="0.9"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus-collector-federation",quantile="0.99"} NaN
prometheus_target_sync_length_seconds_sum{scrape_job="prometheus-collector-federation"} 0.002051667
prometheus_target_sync_length_seconds_count{scrape_job="prometheus-collector-federation"} 9
prometheus_target_sync_length_seconds{scrape_job="prometheus-frontend",quantile="0.01"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus-frontend",quantile="0.05"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus-frontend",quantile="0.5"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus-frontend",quantile="0.9"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus-frontend",quantile="0.99"} NaN
prometheus_target_sync_length_seconds_sum{scrape_job="prometheus-frontend"} 0.000611882
prometheus_target_sync_length_seconds_count{scrape_job="prometheus-frontend"} 9
# HELP prometheus_treecache_watcher_goroutines The current number of watcher goroutines.
# TYPE prometheus_treecache_watcher_goroutines gauge
prometheus_treecache_watcher_goroutines 0
# HELP prometheus_treecache_zookeeper_failures_total The total number of ZooKeeper failures.
# TYPE prometheus_treecache_zookeeper_failures_total counter
prometheus_treecache_zookeeper_failures_total 0