Prometheus routine dump, GH issue 10257

gh_prometheus_10257_routine_dump_01

goroutine 2858169 [running]:
runtime/pprof.writeGoroutineStacks({0x7fc23ed13bb0, 0xc0314b8820})
	/usr/local/go/src/runtime/pprof/pprof.go:693 +0x70
runtime/pprof.writeGoroutine({0x7fc23ed13bb0, 0xc0314b8820}, 0xc00085e800)
	/usr/local/go/src/runtime/pprof/pprof.go:682 +0x2b
runtime/pprof.(*Profile).WriteTo(0x2974720, {0x7fc23ed13bb0, 0xc0314b8820}, 0xc)
	/usr/local/go/src/runtime/pprof/pprof.go:331 +0x14b
net/http/pprof.handler.ServeHTTP({0xc0279c60fd, 0x8281e7}, {0x7fc3884310e0, 0xc0314b8820}, 0x8)
	/usr/local/go/src/net/http/pprof/pprof.go:253 +0x49a
net/http/pprof.Index({0x7fc3884310e0, 0xc0314b8820}, 0xc0374e9b00)
	/usr/local/go/src/net/http/pprof/pprof.go:371 +0x12e
github.com/prometheus/prometheus/web.serveDebug({0x7fc3884310e0, 0xc0314b8820}, 0xc0374e9b00)
	/app/web/web.go:532 +0x2c5
net/http.HandlerFunc.ServeHTTP(0x7fc3884310e0, {0x7fc3884310e0, 0xc0314b8820}, 0xc00085e800)
	/usr/local/go/src/net/http/server.go:2047 +0x2f
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1({0x7fc3884310e0, 0xc0314b87d0}, 0xc0374e9b00)
	/go/pkg/mod/github.com/prometheus/[email protected]/prometheus/promhttp/instrument_server.go:198 +0xa3
net/http.HandlerFunc.ServeHTTP(0x354ea00, {0x7fc3884310e0, 0xc0314b87d0}, 0xc0314d8840)
	/usr/local/go/src/net/http/server.go:2047 +0x2f
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2({0x7fc3884310e0, 0xc0314b87d0}, 0xc0374e9b00)
	/go/pkg/mod/github.com/prometheus/[email protected]/prometheus/promhttp/instrument_server.go:76 +0xa2
net/http.HandlerFunc.ServeHTTP(0x35899d0, {0x7fc3884310e0, 0xc0314b87d0}, 0x0)
	/usr/local/go/src/net/http/server.go:2047 +0x2f
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1({0x35899d0, 0xc0314b8780}, 0xc0374e9b00)
	/go/pkg/mod/github.com/prometheus/[email protected]/prometheus/promhttp/instrument_server.go:101 +0x92
github.com/prometheus/prometheus/web.setPathWithPrefix.func1.1({0x35899d0, 0xc0314b8780}, 0xc0374e9a00)
	/app/web/web.go:1173 +0x290
github.com/prometheus/common/route.(*Router).handle.func1({0x35899d0, 0xc0314b8780}, 0xc0374e9900, {0xc0314c3020, 0x1, 0xc00085e800})
	/go/pkg/mod/github.com/prometheus/[email protected]/route/route.go:83 +0x2ae
github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc0003b9440, {0x35899d0, 0xc0314b8780}, 0xc0374e9900)
	/go/pkg/mod/github.com/julienschmidt/[email protected]/router.go:387 +0x84b
github.com/prometheus/common/route.(*Router).ServeHTTP(0x7fc0b9b70f40, {0x35899d0, 0xc0314b8780}, 0x7fc3af275a68)
	/go/pkg/mod/github.com/prometheus/[email protected]/route/route.go:126 +0x26
net/http.(*ServeMux).ServeHTTP(0x40d187, {0x35899d0, 0xc0314b8780}, 0xc0374e9900)
	/usr/local/go/src/net/http/server.go:2425 +0x149
github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5({0x35884d0, 0xc031403340}, 0xc0374e9800)
	/go/pkg/mod/github.com/opentracing-contrib/[email protected]/nethttp/server.go:154 +0x62d
net/http.HandlerFunc.ServeHTTP(0x0, {0x35884d0, 0xc031403340}, 0xc05ba3d3b0)
	/usr/local/go/src/net/http/server.go:2047 +0x2f
github.com/prometheus/prometheus/web.withStackTracer.func1({0x35884d0, 0xc031403340}, 0xc0314e4600)
	/app/web/web.go:110 +0x97
net/http.HandlerFunc.ServeHTTP(0x7fffa8e39072, {0x35884d0, 0xc031403340}, 0x90)
	/usr/local/go/src/net/http/server.go:2047 +0x2f
github.com/prometheus/exporter-toolkit/web.(*webHandler).ServeHTTP(0xc0005daa40, {0x35884d0, 0xc031403340}, 0x4668ce)
	/go/pkg/mod/github.com/prometheus/[email protected]/web/handler.go:101 +0x852
net/http.serverHandler.ServeHTTP({0xc0314d8660}, {0x35884d0, 0xc031403340}, 0xc0374e9800)
	/usr/local/go/src/net/http/server.go:2879 +0x43b
net/http.(*conn).serve(0xc0314a7180, {0x35a0d38, 0xc000f2f740})
	/usr/local/go/src/net/http/server.go:1930 +0xb08
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3034 +0x4e8

goroutine 1 [chan receive, 892 minutes]:
github.com/oklog/run.(*Group).Run(0xc000eb1ac8)
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:43 +0x7c
main.main()
	/app/cmd/prometheus/main.go:1063 +0x8bf3

goroutine 23 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc000158000)
	/go/pkg/mod/[email protected]/stats/view/worker.go:276 +0xb9
created by go.opencensus.io/stats/view.init.0
	/go/pkg/mod/[email protected]/stats/view/worker.go:34 +0x92

goroutine 87 [select]:
github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).sender(0xc00084dc20)
	/app/discovery/legacymanager/manager.go:235 +0xdd
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).Run
	/app/discovery/legacymanager/manager.go:142 +0x5f

goroutine 102 [select]:
github.com/prometheus/prometheus/util/logging.(*Deduper).run(0xc0005cf680)
	/app/util/logging/dedupe.go:75 +0xee
created by github.com/prometheus/prometheus/util/logging.Dedupe
	/app/util/logging/dedupe.go:61 +0x10f

goroutine 109 [select]:
github.com/prometheus/prometheus/storage/remote.(*WriteStorage).run(0xc00084d9a0)
	/app/storage/remote/write.go:107 +0xdd
created by github.com/prometheus/prometheus/storage/remote.NewWriteStorage
	/app/storage/remote/write.go:99 +0x485

goroutine 80 [syscall, 892 minutes]:
os/signal.signal_recv()
	/usr/local/go/src/runtime/sigqueue.go:169 +0x98
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
	/usr/local/go/src/os/signal/signal.go:151 +0x2c

goroutine 86 [select]:
github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).sender(0xc00084dae0)
	/app/discovery/legacymanager/manager.go:235 +0xdd
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).Run
	/app/discovery/legacymanager/manager.go:142 +0x5f

goroutine 273 [select, 892 minutes]:
main.main.func6()
	/app/cmd/prometheus/main.go:774 +0xb1
github.com/oklog/run.(*Group).Run.func1({0xc000860380, 0xc0008d8530})
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:38 +0x2f
created by github.com/oklog/run.(*Group).Run
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:37 +0x22f

goroutine 274 [chan receive, 892 minutes]:
github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).Run(0xc00084dae0)
	/app/discovery/legacymanager/manager.go:143 +0x7c
main.main.func8()
	/app/cmd/prometheus/main.go:794 +0x38
github.com/oklog/run.(*Group).Run.func1({0xc00072d740, 0xc0002082a0})
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:38 +0x2f
created by github.com/oklog/run.(*Group).Run
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:37 +0x22f

goroutine 275 [chan receive, 892 minutes]:
github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).Run(0xc00084dc20)
	/app/discovery/legacymanager/manager.go:143 +0x7c
main.main.func10()
	/app/cmd/prometheus/main.go:808 +0x38
github.com/oklog/run.(*Group).Run.func1({0xc00072d770, 0xc0002082e0})
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:38 +0x2f
created by github.com/oklog/run.(*Group).Run
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:37 +0x22f

goroutine 276 [select, 1 minutes]:
github.com/prometheus/prometheus/scrape.(*Manager).Run(0xc0003b8600, 0xc000414300)
	/app/scrape/manager.go:150 +0xc5
main.main.func12()
	/app/cmd/prometheus/main.go:828 +0x85
github.com/oklog/run.(*Group).Run.func1({0xc000860400, 0xc000208340})
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:38 +0x2f
created by github.com/oklog/run.(*Group).Run
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:37 +0x22f

goroutine 277 [select, 889 minutes]:
main.main.func14()
	/app/cmd/prometheus/main.go:853 +0x145
github.com/oklog/run.(*Group).Run.func1({0xc0002005a0, 0xc0008d8560})
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:38 +0x2f
created by github.com/oklog/run.(*Group).Run
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:37 +0x22f

goroutine 278 [chan receive, 889 minutes]:
main.main.func16()
	/app/cmd/prometheus/main.go:898 +0x2d8
github.com/oklog/run.(*Group).Run.func1({0xc000200600, 0xc0008d8570})
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:38 +0x2f
created by github.com/oklog/run.(*Group).Run
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:37 +0x22f

goroutine 279 [chan receive, 889 minutes]:
github.com/prometheus/prometheus/rules.(*Manager).Run(...)
	/app/rules/manager.go:933
main.main.func18()
	/app/cmd/prometheus/main.go:911 +0x4b
github.com/oklog/run.(*Group).Run.func1({0xc000202540, 0xc0008d8580})
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:38 +0x2f
created by github.com/oklog/run.(*Group).Run
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:37 +0x22f

goroutine 280 [chan receive, 889 minutes]:
main.main.func20()
	/app/cmd/prometheus/main.go:963 +0xb56
github.com/oklog/run.(*Group).Run.func1({0xc0000be870, 0xc00072d7a0})
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:38 +0x2f
created by github.com/oklog/run.(*Group).Run
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:37 +0x22f

goroutine 281 [select, 892 minutes]:
github.com/prometheus/prometheus/web.(*Handler).Run(0xc000c9e500, {0x35a0c90, 0xc0005cf780}, {0x357dbb0, 0xc000d5fce0}, {0x7fffa8e39072, 0x2a})
	/app/web/web.go:626 +0x7f9
main.main.func24()
	/app/cmd/prometheus/main.go:1031 +0x38
github.com/oklog/run.(*Group).Run.func1({0xc000860440, 0xc0008d85a0})
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:38 +0x2f
created by github.com/oklog/run.(*Group).Run
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:37 +0x22f

goroutine 282 [select]:
github.com/prometheus/prometheus/notifier.(*Manager).Run(0xc000c94d00, 0xc0004143c0)
	/app/notifier/notifier.go:307 +0xc5
main.main.func26()
	/app/cmd/prometheus/main.go:1054 +0x85
github.com/oklog/run.(*Group).Run.func1({0xc000860480, 0xc0008d85b0})
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:38 +0x2f
created by github.com/oklog/run.(*Group).Run
	/go/pkg/mod/github.com/oklog/[email protected]/group.go:37 +0x22f

goroutine 373 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388496118, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc000d42c00, 0x2, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000d42c00)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000d42c00)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000d5fc98)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000d5fc98)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
golang.org/x/net/netutil.(*limitListener).Accept(0xc000d7b320)
	/go/pkg/mod/golang.org/x/[email protected]/netutil/listen.go:48 +0x3e
github.com/mwitkow/go-conntrack.(*connTrackListener).Accept(0xc000d5fce0)
	/go/pkg/mod/github.com/mwitkow/[email protected]/listener_wrapper.go:100 +0x6c
net/http.(*Server).Serve(0xc000f32000, {0x357dbb0, 0xc000d5fce0})
	/usr/local/go/src/net/http/server.go:3002 +0x394
github.com/prometheus/exporter-toolkit/web.Serve({0x357dbb0, 0xc000d5fce0}, 0xc000f32000, {0x7fffa8e39072, 0x2a}, {0x3547580, 0xc000d3e190})
	/go/pkg/mod/github.com/prometheus/[email protected]/web/tls_config.go:232 +0x517
github.com/prometheus/prometheus/web.(*Handler).Run.func2()
	/app/web/web.go:623 +0x45
created by github.com/prometheus/prometheus/web.(*Handler).Run
	/app/web/web.go:622 +0x779

goroutine 391 [select, 56 minutes]:
github.com/prometheus/prometheus/tsdb/wal.(*WAL).run(0xc000ea79e0)
	/app/tsdb/wal/wal.go:332 +0xaa
created by github.com/prometheus/prometheus/tsdb/wal.NewSize
	/app/tsdb/wal/wal.go:301 +0x2cf

goroutine 2713669 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc057582ea0, 0xc020792b40)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2563 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0005f68f0, 0x0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1463 [chan receive, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3208)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000e0a738, {0x354b380, 0xc059f845d0}, 0x1, 0xc059db5b60)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00005a018, 0x3b9aca00, 0x0, 0xc0, 0xc000e0a788)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc059acab00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2683271 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc059affc70, 0xc057f717a0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2713324 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc029d28270, 0xc04f663590)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1511 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 2731652 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc02360a270, 0xc02cf0dd80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1784011 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba93490, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05f180f00, 0xc028b5f000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05f180f00, {0xc028b5f000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05f180f00, {0xc028b5f000, 0xc06d2d8f00, 0xc05ac72f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc02fcd9078, {0xc028b5f000, 0xc05614c1a0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc02a162a20, {0xc028b5f000, 0xc05b746000, 0xc05ac72d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc06afb4f00)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc06afb4f00, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc02a162a20)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1784393 [select]:
net/http.(*persistConn).writeLoop(0xc0027939e0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1371 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1284 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc059ad0240)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:204 +0xa7
created by k8s.io/client-go/util/workqueue.newQueue
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:62 +0x1af

goroutine 2552980 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01df0ca90, 0xc019104140)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1352 [sync.Cond.Wait, 9 minutes]:
sync.runtime_notifyListWait(0xc059d92ac8, 0x59)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc014bd6e00)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d92aa0, 0xc059dbb230)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc059dc8bd0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3208)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059f840f0}, 0x1, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059dc8c38, 0x3b9aca00, 0x0, 0x0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc059dc8bd0, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc059d92280, 0xc055949860)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:448 +0x1d6f

goroutine 2420 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0641a81a0, 0xc059b4d980)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 381 [select]:
github.com/prometheus/prometheus/tsdb.(*DB).run(0xc000ec8680)
	/app/tsdb/db.go:798 +0x18e
created by github.com/prometheus/prometheus/tsdb.open
	/app/tsdb/db.go:748 +0xc16

goroutine 2493 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0005f6820, 0xc06d252000)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2810 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916b780, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0106b2280, 0xc0106c8000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0106b2280, {0xc0106c8000, 0x3c7b, 0x3c7b})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0106b2280, {0xc0106c8000, 0x2c57, 0xc00d44ce20})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc06d2635d8, {0xc0106c8000, 0xc0106c901f, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc05061c1b0, {0xc0106c8000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc00d4d73f8, {0x3544be0, 0xc05061c1b0})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc00d4d7180, {0x7fc0b9166040, 0xc01063bbc0}, 0x2c5c)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc00d4d7180, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc00d4d7180, {0xc00b134000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc00d452d20, {0xc009077c40, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc00d452d20}, {0xc009077c40, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc009077c40, 0x9, 0xc001c75f50}, {0x35449e0, 0xc00d452d20})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc009077c00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc007acff98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00d09c300)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2552527 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc03f721110, 0xc028ebebc0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1462 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc059acaa80)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1561 [sync.Cond.Wait, 9 minutes]:
sync.runtime_notifyListWait(0xc057d77f50, 0x97)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x24e9980)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/util/workqueue.(*Type).Get(0xc059ad0a80)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:151 +0x9e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc059acb180, {0x35a0c90, 0xc059b9cd00}, 0x82b825)
	/app/discovery/kubernetes/endpoints.go:154 +0x6e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1()
	/app/discovery/kubernetes/endpoints.go:145 +0x53
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:144 +0x2c5

goroutine 2849362 [select, 3 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc024ef3c80, 0xc062981e00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0xc04d2133e0, 0xc04d2133e0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 1354 [chan receive, 889 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc059acb180, {0x35a0c90, 0xc059b9cd00}, 0xc059b4da40)
	/app/discovery/kubernetes/endpoints.go:150 +0x2e5
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc059acb180})
	/app/discovery/kubernetes/kubernetes.go:580 +0x75
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:578 +0x8cf

goroutine 2367718 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc014887e10, 0xc00f5f8940)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2641716 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916f440, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0539fed80, 0xc02f3cb000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0539fed80, {0xc02f3cb000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0539fed80, {0xc02f3cb000, 0xc06d2d8640, 0xc002bdff68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc02e7ea000, {0xc02f3cb000, 0xc04838a9c0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc03485f7a0, {0xc02f3cb000, 0xc001ee7140, 0xc002bdfd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc00d452120)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc00d452120, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc03485f7a0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1445 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1446 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f53420, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc036838840}, 0xc055eddce8, 0xc05b3e4720, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f53420, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059dbe6e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059dbc640, {0x354b360, 0xc059dbe640}, 0x1, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f53420, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2734984 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc03dfd3ba0, 0xc0124aae80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1452 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f53500, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc045d07b00}, 0xc055edbce8, 0xc05b3e5800, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f53500, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059dbe820)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059dbc780, {0x354b360, 0xc059dbe780}, 0x1, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f53500, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 695258 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388495808, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc056974b00, 0xc050ed6000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc056974b00, {0xc050ed6000, 0x81e, 0x81e})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc056974b00, {0xc050ed6000, 0x819, 0xc04bff9140})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc04586b300, {0xc050ed6000, 0xc050ed6000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc022293278, {0xc050ed6000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc020f75078, {0x3544be0, 0xc022293278})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc020f74e00, {0x7fc0b9166040, 0xc0206cbe80}, 0x81e)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc020f74e00, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc020f74e00, {0xc03d489000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc03d356480, {0xc0217dd620, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc03d356480}, {0xc0217dd620, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0217dd620, 0x9, 0xc001066690}, {0x35449e0, 0xc03d356480})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0217dd5e0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc002ce0f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc020e33b00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 1466 [select, 3 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc059acad00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1786680 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01a21dd40, 0xc0260cb980)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2851661 [select]:
net/http.(*persistConn).writeLoop(0xc0214d4900)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1364 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d800e0, 0xc000504f40)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1366 [select, 3 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f52d20, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc03a66a300}, 0xc055d45ce8, 0xc059fdb8c0, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f52d20, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059abbbd0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059a9bd80, {0x354b360, 0xc059abbb30}, 0x1, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f52d20, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2425 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0641a8820, 0xc06d221000)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1589 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 2686922 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9180420, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02895e200, 0xc05bf1c900, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02895e200, {0xc05bf1c900, 0x81e, 0x81e})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02895e200, {0xc05bf1c900, 0x7de, 0xc0176f7dc0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc04a8a4ac0, {0xc05bf1c900, 0xc05bf1c93b, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc017367c20, {0xc05bf1c900, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc015cf7778, {0x3544be0, 0xc017367c20})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc015cf7500, {0x7fc0b9166040, 0xc01ba74540}, 0x7e3)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc015cf7500, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc015cf7500, {0xc017771000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc052955740, {0xc023faaf20, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc052955740}, {0xc023faaf20, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc023faaf20, 0x9, 0xc00104cc60}, {0x35449e0, 0xc052955740})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc023faaee0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc055ee3f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc014a9f680)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 1360 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f52c40, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc02dc34280}, 0xc055f15ce8, 0xc059fdb500, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f52c40, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059abba90)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059a9bd00, {0x354b360, 0xc059abb9f0}, 0x1, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f52c40, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 704696 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9187968, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05699b480, 0xc002f40000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05699b480, {0xc002f40000, 0x3cd5, 0x3cd5})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05699b480, {0xc002f40000, 0x3cd0, 0xc06cebd860})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc01cb959a0, {0xc002f40000, 0xc002f40000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc020864360, {0xc002f40000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0624e8cf8, {0x3544be0, 0xc020864360})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0624e8a80, {0x7fc0b9166040, 0xc018348240}, 0x3cd5)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0624e8a80, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0624e8a80, {0xc02f3ca000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc0578d9ec0, {0xc05867a120, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc0578d9ec0}, {0xc05867a120, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc05867a120, 0x9, 0xc001845320}, {0x35449e0, 0xc0578d9ec0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc05867a0e0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc051011f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc002f9e000)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 1350 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc059ad0a80)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:204 +0xa7
created by k8s.io/client-go/util/workqueue.newQueue
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:62 +0x1af

goroutine 2419 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06d2ec9c0, 0xc06d221200)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1483 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 2554359 [select]:
net/http.(*persistConn).writeLoop(0xc0106db560)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1744047 [select]:
net/http.(*persistConn).writeLoop(0xc029eaf8c0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1786678 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01a21dc70, 0xc0260569f0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2423 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0641a85b0, 0xc06d221000)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1345 [sync.Cond.Wait, 9 minutes]:
sync.runtime_notifyListWait(0xc059d928e8, 0x59)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc01cb076a0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d928c0, 0xc059dba960)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc059db7c20)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3e40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059d9d980}, 0x1, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059db7c88, 0x3b9aca00, 0x0, 0x80, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc059db7c20, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc059d920a0, 0xc056126d00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:448 +0x1d6f

goroutine 2850742 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916b2f8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02efd7280, 0xc02fc66000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02efd7280, {0xc02fc66000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02efd7280, {0xc02fc66000, 0xc06d2d8c80, 0xc002bdbf68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0018545e8, {0xc02fc66000, 0xc01bde6d00, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc00a1db0e0, {0xc02fc66000, 0xc025a8d9e0, 0xc002bdbd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc02e371c80)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc02e371c80, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc00a1db0e0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1410 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d803f0, 0xc0562f87b0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1548 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 2555625 [select]:
net/http.(*persistConn).writeLoop(0xc00dc238c0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2731683 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01bb72820, 0xc06d258600)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1744129 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916b210, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05e12b580, 0xc004d54000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05e12b580, {0xc004d54000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05e12b580, {0xc004d54000, 0xc06d2d9400, 0xc00c30bf68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc052dd2b50, {0xc004d54000, 0xc05ee54680, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0196fa6c0, {0xc004d54000, 0xc04ddf3f80, 0xc00c30bd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc04e195920)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc04e195920, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0196fa6c0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1457 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 808 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 1475 [chan receive, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3208)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc055324738, {0x354b380, 0xc059f846f0}, 0x1, 0xc059db5da0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x602419d0, 0x3b9aca00, 0x0, 0x46, 0xc055324788)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc059acb280)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2686305 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01df0c270, 0xc06cceba00)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2552526 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc03f720f70, 0xc0231d1d40)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2714235 [select]:
net/http.(*persistConn).writeLoop(0xc01a2db8c0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 814 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 1292 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0561abea0, 0xc059a8e960)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1286 [sync.Cond.Wait, 9 minutes]:
sync.runtime_notifyListWait(0xc056197b08, 0x59)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc041383120)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc056197ae0, 0xc059abe9c0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc057bdee10)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bcbbd0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059aa4f00}, 0x1, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc057bdee78, 0x3b9aca00, 0x0, 0x0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc057bdee10, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc056197860, 0xc059b4d860)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:448 +0x1d6f

goroutine 1327 [chan receive, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bcbbd0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc055f09738, {0x354b380, 0xc059aa53e0}, 0x1, 0xc059a8f1a0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x55bfc7b0, 0x3b9aca00, 0x0, 0x46, 0xc055f09788)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc059aca580)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2357 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06d2eda00, 0xc06cb23080)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1465 [chan receive, 3 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3208)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc055d18738, {0x354b380, 0xc059f84600}, 0x1, 0xc059db5bc0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x3544e60, 0x3b9aca00, 0x0, 0xb0, 0xc055d18788)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc059acad00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1473 [chan receive, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3208)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc055d17f38, {0x354b380, 0xc059f846c0}, 0x1, 0xc059db5d40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0xb0, 0xc055d17f88)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc059acb200)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2372021 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc021875860, 0x0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1346 [sync.Cond.Wait, 889 minutes]:
sync.runtime_notifyListWait(0xc059d92988, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d92960, 0xc059dbac50)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc059dc81b0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3e40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059d9dbf0}, 0x1, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059dc8218, 0x3b9aca00, 0x0, 0x0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc059dc81b0, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc059d921e0, 0xc056126b00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:449 +0x1e2f

goroutine 1365 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1456 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d805b0, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2555624 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388495e60, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc026e8bb00, 0xc01bd1d000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc026e8bb00, {0xc01bd1d000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc026e8bb00, {0xc01bd1d000, 0xc06d2d8f00, 0xc05ac76f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0304d95a8, {0xc01bd1d000, 0xc010ac7380, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc00dc238c0, {0xc01bd1d000, 0xc025ea6420, 0xc05ac76d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc02e565f20)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc02e565f20, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc00dc238c0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1481 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 1285 [sync.Cond.Wait, 2 minutes]:
sync.runtime_notifyListWait(0xc056197a68, 0x2a7)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc0205450e0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc056197a40, 0xc059abe6d0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc057bde900)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bcbbd0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059aa4c90}, 0x1, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc057bde968, 0x3b9aca00, 0x0, 0x80, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc057bde900, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc056197900, 0x0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:447 +0x1caf

goroutine 1325 [chan receive, 2 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bcbbd0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc055174738, {0x354b380, 0xc059aa53b0}, 0x1, 0xc059a8f140)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0xb0, 0xc055174788)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc059aca500)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1467 [chan receive, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3208)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc055d13f38, {0x354b380, 0xc059f84630}, 0x1, 0xc059db5c20)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc055bb1a40, 0x3b9aca00, 0x0, 0x38, 0xc055d13f88)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc059acad80)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1383 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1344 [sync.Cond.Wait, 9 minutes]:
sync.runtime_notifyListWait(0xc059d92848, 0x59)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc01cb07700)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d92820, 0xc059dba670)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc059db7710)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3e40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059d9d710}, 0x1, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059db7778, 0x3b9aca00, 0x0, 0x40, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc059db7710, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc059d92140, 0xc056127000)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:447 +0x1caf

goroutine 663 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388495d78, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc059cd5680, 0xc056360000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc059cd5680, {0xc056360000, 0xba68, 0xba68})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc059cd5680, {0xc056360000, 0xc056360005, 0xa1})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc04daa1a98, {0xc056360000, 0x6, 0xc055cfb7f0})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc017992f30, {0xc056360000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc055eb8278, {0x3544be0, 0xc017992f30})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc055eb8000, {0x354b880, 0xc04daa1a98}, 0xba68)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc055eb8000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc055eb8000, {0xc055efc000, 0x1000, 0xc055cfbcb0})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc055ef4b40, {0xc05050cc80, 0x9, 0x87})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc055ef4b40}, {0xc05050cc80, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc05050cc80, 0x9, 0xc001891f50}, {0x35449e0, 0xc055ef4b40})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc05050cc40)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc055cfbf98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc05066c600)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2641717 [select]:
net/http.(*persistConn).writeLoop(0xc03485f7a0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1416 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d80380, 0x0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1372 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f52e00, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc01df44c40}, 0xc055ec1ce8, 0xc05ba7ff80, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f52e00, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059abbd10)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059a9bdc0, {0x354b360, 0xc059abbc70}, 0x1, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f52e00, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1417 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1513 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 1424 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f53340, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc03319c0c0}, 0xc04dcffce8, 0xc05a0c8660, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f53340, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059dbe550)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059dbc500, {0x354b360, 0xc059dbe4b0}, 0x1, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f53340, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1339 [sync.Cond.Wait, 3 minutes]:
sync.runtime_notifyListWait(0xc059d927a8, 0x25f)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc036b778c0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d92780, 0xc059dba380)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc059db7200)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3e40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059d9d4a0}, 0x1, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059db7268, 0x3b9aca00, 0x0, 0x0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc059db7200, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc059d92000, 0xc059acaa00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:449 +0x1e2f

goroutine 1422 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d80460, 0xc055f0f7a8)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2552762 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916b4c8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05d163500, 0xc05fb6e900, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05d163500, {0xc05fb6e900, 0x820, 0x820})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05d163500, {0xc05fb6e900, 0x81b, 0xc018b47360})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc01a7545f8, {0xc05fb6e900, 0xc05fb6e900, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc015623110, {0xc05fb6e900, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc018c25778, {0x3544be0, 0xc015623110})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc018c25500, {0x7fc0b9166040, 0xc024806640}, 0x820)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc018c25500, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc018c25500, {0xc018b57000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc059a04a80, {0xc05b99e9e0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc059a04a80}, {0xc05b99e9e0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc05b99e9e0, 0x9, 0xc0015959b0}, {0x35449e0, 0xc059a04a80})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc05b99e9a0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc003086f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc0180abb00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 1411 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1388 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d80310, 0x402c000000000000)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1370 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d801c0, 0xc055d157a8)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1569 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 1412 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f53180, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc03952e400}, 0xc055f59ce8, 0xc05a2d4f00, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f53180, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059dbe2d0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059dbc240, {0x354b360, 0xc059dbe230}, 0x1, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f53180, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1470 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc059acaf80)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1347 [chan receive, 889 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc059acaf00, {0x35a0c90, 0xc059b9cc80}, 0xc059b4d9e0)
	/app/discovery/kubernetes/endpoints.go:150 +0x2e5
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc059acaf00})
	/app/discovery/kubernetes/kubernetes.go:580 +0x75
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:578 +0x8cf

goroutine 1471 [chan receive, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3208)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc055f0ff38, {0x354b380, 0xc059f84690}, 0x1, 0xc059db5ce0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0504554a0, 0x3b9aca00, 0x0, 0xb8, 0xc055f0ff88)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc059acb000)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1423 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1389 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1384 [select, 5 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f52fc0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc023d7e840}, 0xc05516dce8, 0xc05a0c8f00, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f52fc0, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059dbe000)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059dbc000, {0x354b360, 0xc059abbf40}, 0x1, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f52fc0, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1571 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 2715954 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b90551b0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc036082d80, 0xc05106c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc036082d80, {0xc05106c000, 0x81e, 0x81e})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc036082d80, {0xc05106c000, 0x819, 0xc01b56fb80})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0352044e8, {0xc05106c000, 0xc05106c000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0173d31b8, {0xc05106c000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc01b6765f8, {0x3544be0, 0xc0173d31b8})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc01b676380, {0x7fc0b9166040, 0xc01b883100}, 0x81e)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc01b676380, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc01b676380, {0xc0265ae000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc047a3db00, {0xc06db88ac0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc047a3db00}, {0xc06db88ac0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc06db88ac0, 0x9, 0xc0015ee600}, {0x35449e0, 0xc047a3db00})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc06db88a80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc002be0f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01b650c00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 1353 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc059d92b68, 0x15ba)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc016bd6960)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d92b40, 0xc059dbb520)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc059dc90e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3208)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059f84360}, 0x1, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059dc9148, 0x3b9aca00, 0x0, 0x0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc059dc90e0, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc059d923c0, 0xc055f3d768)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:449 +0x1e2f

goroutine 1418 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f53260, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc01b834cc0}, 0xc055edfce8, 0xc05a0c8240, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f53260, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059dbe410)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059dbc3c0, {0x354b360, 0xc059dbe370}, 0x1, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f53260, 0xc059a8f9e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1474 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc059acb200)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1294 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f529a0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc02a54f600}, 0xc04dcf9ce8, 0xc059fda8a0, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f529a0, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059abb360)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059a9b980, {0x354b360, 0xc059abb2c0}, 0x1, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f529a0, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1293 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 2848972 [select, 3 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc024ef3380, 0xc0356e9100)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a16a0, 0xc02d72b500)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 2554219 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9054988, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0579feb00, 0xc00ea7c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0579feb00, {0xc00ea7c000, 0x3d13, 0x3d13})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0579feb00, {0xc00ea7c000, 0x3d0e, 0xc012232640})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc023184df8, {0xc00ea7c000, 0xc00ea7c000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc022292f00, {0xc00ea7c000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc02e009e78, {0x3544be0, 0xc022292f00})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc02e009c00, {0x7fc0b9166040, 0xc04bb52240}, 0x3d13)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc02e009c00, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc02e009c00, {0xc015ccc000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc064352ae0, {0xc02a09a3c0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc064352ae0}, {0xc02a09a3c0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc02a09a3c0, 0x9, 0xc0010660f0}, {0x35449e0, 0xc064352ae0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc02a09a380)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc055d68f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00e0fca80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 1377 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 695987 [select]:
net/http.(*persistConn).writeLoop(0xc0532ee480)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1382 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d80230, 0xc00006bf58)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1288 [chan receive, 889 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc059aca480, {0x35a0c90, 0xc059b9cb00}, 0xc059b4d860)
	/app/discovery/kubernetes/endpoints.go:150 +0x2e5
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc059aca480})
	/app/discovery/kubernetes/kubernetes.go:580 +0x75
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:578 +0x8cf

goroutine 1287 [sync.Cond.Wait, 2 minutes]:
sync.runtime_notifyListWait(0xc056197ba8, 0xeee)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc024191760)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc056197b80, 0xc059abecb0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc057bdf320)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bcbbd0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059aa5170}, 0x1, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc057bdf388, 0x3b9aca00, 0x0, 0x0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc057bdf320, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0561979a0, 0xc055f3b7b8)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:449 +0x1e2f

goroutine 2847463 [sync.Cond.Wait, 3 minutes]:
sync.runtime_notifyListWait(0xc03a69e1c8, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc03a69e1b8)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc03a69e1b0, {0xc034ebb600, 0x200, 0x200})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc034ebb600, 0x217c472, 0xc058651cf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc03a4d9180)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc03a4d9180)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc03a4d9180, {0x2748340, 0xc03a6414e8})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc03a6674a0, {0xc03a670400, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc03a660690, 0xc058651e38, {0x357ac00, 0xc03a66a340})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc03a631ea0)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc03a66a300)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2507784 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916b3e0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05a9f7d80, 0xc01f0f7000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05a9f7d80, {0xc01f0f7000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05a9f7d80, {0xc01f0f7000, 0xc06d2d8b40, 0xc064026f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0128d6c50, {0xc01f0f7000, 0xc05baf3d40, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc02808fb00, {0xc01f0f7000, 0xc05b077140, 0xc064026d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc02eced920)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc02eced920, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc02808fb00)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1376 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d802a0, 0xc055ff57b0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1781544 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba93008, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02c67a980, 0xc004df4000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02c67a980, {0xc004df4000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02c67a980, {0xc004df4000, 0xc06d2d8c80, 0xc055d57f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00e4ee4b8, {0xc004df4000, 0xc029b94ea0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc035fca7e0, {0xc004df4000, 0xc001c45740, 0xc055d57d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc053543380)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc053543380, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc035fca7e0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1206061 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916e790, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0411bb280, 0xc04719d000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0411bb280, {0xc04719d000, 0x162a, 0x162a})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0411bb280, {0xc04719d000, 0x15ea, 0xc01cb07040})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc026ad53f0, {0xc04719d000, 0xc04719d03b, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc01725c810, {0xc04719d000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc01a93be78, {0x3544be0, 0xc01725c810})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc01a93bc00, {0x7fc0b9166040, 0xc0069d4dc0}, 0x15ef)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc01a93bc00, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc01a93bc00, {0xc031471000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc05b6a4ae0, {0xc01417c820, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc05b6a4ae0}, {0xc01417c820, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc01417c820, 0x9, 0xc001bcc7b0}, {0x35449e0, 0xc05b6a4ae0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc01417c7e0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00ed19f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01c438480)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 1359 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1461 [chan receive, 1 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3208)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0562f6f38, {0x354b380, 0xc059f845a0}, 0x1, 0xc059db5b00)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0580d64e0, 0x3b9aca00, 0x0, 0xa0, 0xc0562f6f88)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc059acaa80)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2714236 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b91254f8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc055da9580, 0xc01988c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc055da9580, {0xc01988c000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc055da9580, {0xc01988c000, 0xc06d2d8640, 0xc00cc9df68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00cfac970, {0xc01988c000, 0xc000eb6340, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc028138ea0, {0xc01988c000, 0xc05f4c0f00, 0xc00cc9dd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc03d9c1860)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03d9c1860, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc028138ea0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1476 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc059acb280)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1464 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc059acab00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 3694 [select]:
net/http.(*persistConn).writeLoop(0xc0127d18c0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1314 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0561abe30, 0xc0504636d5)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1315 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 692152 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06946be10, 0xc03f55cf80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1378 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f52ee0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc01fb01340}, 0xc055169ce8, 0xc0580d7c80, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f52ee0, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059abbea0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059a9bec0, {0x354b360, 0xc059abbe00}, 0x1, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f52ee0, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1477 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 1823 [sync.Cond.Wait, 3 minutes]:
sync.runtime_notifyListWait(0xc0582d89d0, 0x170)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x24e9980)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/util/workqueue.(*Type).Get(0xc0580d70e0)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:151 +0x9e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc057c73e80, {0x35a0c90, 0xc059b9d900}, 0x82b825)
	/app/discovery/kubernetes/endpoints.go:154 +0x6e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1()
	/app/discovery/kubernetes/endpoints.go:145 +0x53
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:144 +0x2c5

goroutine 2857664 [select]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc0161de600, 0xc070343100)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a16a0, 0xc021b724c0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 1450 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d804d0, 0x4133aa0000000000)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1451 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1821 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc057d77d10, 0x233)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x24e9980)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/util/workqueue.(*Type).Get(0xc059ad0840)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:151 +0x9e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc059acaa00, {0x35a0c90, 0xc059b9cb80}, 0x82b825)
	/app/discovery/kubernetes/endpoints.go:154 +0x6e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1()
	/app/discovery/kubernetes/endpoints.go:145 +0x53
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:144 +0x2c5

goroutine 2730500 [select]:
net/http.(*persistConn).writeLoop(0xc01e552d80)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1351 [sync.Cond.Wait, 9 minutes]:
sync.runtime_notifyListWait(0xc059d92a28, 0x64)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc01c61b1a0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d92a00, 0xc059dbaf40)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc059dc86c0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3e40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059d9de60}, 0x1, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059dc8728, 0x3b9aca00, 0x0, 0x80, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc059dc86c0, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc059d92320, 0xc056921768)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:447 +0x1caf

goroutine 1472 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc059acb000)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2713160 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0005535f0, 0xc0077da900)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2638833 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba93a00, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05e12af00, 0xc012b5c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05e12af00, {0xc012b5c000, 0x3cd9, 0x3cd9})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05e12af00, {0xc012b5c000, 0x3cd4, 0xc00e983e60})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0247216c0, {0xc012b5c000, 0xc012b5c000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0156485b8, {0xc012b5c000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc00faf0978, {0x3544be0, 0xc0156485b8})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc00faf0700, {0x7fc0b9166040, 0xc021be87c0}, 0x3cd9)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc00faf0700, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc00faf0700, {0xc011288000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc03db73560, {0xc0235822e0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc03db73560}, {0xc0235822e0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0235822e0, 0x9, 0xc00126e5a0}, {0x35449e0, 0xc03db73560})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0235822a0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc058b43f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc010e85680)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2851843 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9054c40, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc013bfd600, 0xc050c83200, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc013bfd600, {0xc050c83200, 0x81c, 0x81c})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc013bfd600, {0xc050c83200, 0x817, 0xc013cbbbe0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc009d87c98, {0xc050c83200, 0xc050c83200, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0411b0768, {0xc050c83200, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0214cfaf8, {0x3544be0, 0xc0411b0768})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0214cf880, {0x7fc0b9166040, 0xc01063b900}, 0x81c)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0214cf880, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0214cf880, {0xc013cc2000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc04b70b4a0, {0xc013c991c0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc04b70b4a0}, {0xc013c991c0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc013c991c0, 0x9, 0xc00175d350}, {0x35449e0, 0xc04b70b4a0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc013c99180)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00755df98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc013c97380)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 1458 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f535e0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc03a5e0780}, 0xc055cf9ce8, 0xc05b3e5e60, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f535e0, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059dbe960)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059dbc880, {0x354b360, 0xc059dbe8c0}, 0x1, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f535e0, 0xc059a8fce0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2842441 [sync.Cond.Wait, 5 minutes]:
sync.runtime_notifyListWait(0xc00b08f848, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc00b08f838)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc00b08f830, {0xc012959a00, 0x200, 0x200})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc012959a00, 0xc0581f5d60, 0xc0581f5cf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc0746a12c0)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc0746a12c0)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc0746a12c0, {0x2748340, 0xc03ba4e330})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc024d57740, {0xc02dfde800, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0127ed630, 0xc059b9cc00, {0x357ac00, 0xc023d7e880})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc01289e2c0)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc023d7e840)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2528 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0001b7520, 0xc05f0db580)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1469 [chan receive, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3208)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc056920738, {0x354b380, 0xc059f84660}, 0x1, 0xc059db5c80)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00005a018, 0x3b9aca00, 0x0, 0xe0, 0xc056920788)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc059acaf80)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2713012 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9187c20, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc033802480, 0xc0252b2000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc033802480, {0xc0252b2000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc033802480, {0xc0252b2000, 0xc06d2d8c80, 0xc000e81f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0123a0130, {0xc0252b2000, 0xc006380d00, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0016dc900, {0xc0252b2000, 0xc009588660, 0xc000e81d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc033ea8cc0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc033ea8cc0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0016dc900)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1338 [sync.Cond.Wait, 9 minutes]:
sync.runtime_notifyListWait(0xc059d92708, 0x59)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc01c61a340)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d926e0, 0xc059dba090)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc059db6cf0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3e40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059d9d230}, 0x1, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059db6d58, 0x3b9aca00, 0x0, 0x0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc059db6cf0, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc056197ea0, 0xc059b4d980)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:448 +0x1d6f

goroutine 1340 [chan receive, 889 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc059acac80, {0x35a0c90, 0xc059b9cc00}, 0xc059b4d980)
	/app/discovery/kubernetes/endpoints.go:150 +0x2e5
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc059acac80})
	/app/discovery/kubernetes/kubernetes.go:580 +0x75
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:578 +0x8cf

goroutine 1788012 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba931d8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02f338400, 0xc01069a000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02f338400, {0xc01069a000, 0x3c7b, 0x3c7b})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02f338400, {0xc01069a000, 0x3c76, 0xc003fd6020})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc001760ed8, {0xc01069a000, 0xc01069a000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc05dd7dda0, {0xc01069a000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc003986978, {0x3544be0, 0xc05dd7dda0})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc003986700, {0x7fc0b9166040, 0xc02160a0c0}, 0x3c7b)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc003986700, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc003986700, {0xc01a219000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc05197ff20, {0xc020ff7380, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc05197ff20}, {0xc020ff7380, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc020ff7380, 0x9, 0xc001253680}, {0x35449e0, 0xc05197ff20})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc020ff7340)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc055d69f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc02998dc80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 699254 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc038503e10, 0xc06c569140)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1744130 [select]:
net/http.(*persistConn).writeLoop(0xc0196fa6c0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1585 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 2372781 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917f9c8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc03ca76400, 0xc00b3a0000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc03ca76400, {0xc00b3a0000, 0x3cd7, 0x3cd7})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc03ca76400, {0xc00b3a0000, 0x3cd2, 0xc036ac0d00})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc020f684a8, {0xc00b3a0000, 0xc00b3a0000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0188336c8, {0xc00b3a0000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0579e4cf8, {0x3544be0, 0xc0188336c8})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0579e4a80, {0x7fc0b9166040, 0xc05a6d0280}, 0x3cd7)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0579e4a80, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0579e4a80, {0xc0248fc000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc056302600, {0xc034ed51c0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc056302600}, {0xc034ed51c0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc034ed51c0, 0x9, 0xc0018255f0}, {0x35449e0, 0xc056302600})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc034ed5180)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00b95af98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc0658e3200)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 1343 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc059ad09c0)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:204 +0xa7
created by k8s.io/client-go/util/workqueue.newQueue
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:62 +0x1af

goroutine 1321 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1782319 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc03dfd3e10, 0xc0589eed00)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1322 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f52b60, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc013826e40}, 0xc0560edce8, 0xc063f2d2c0, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f52b60, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059abb5e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059a9bbc0, {0x354b360, 0xc059abb540}, 0x1, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f52b60, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2529 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0001b7a00, 0xc05f4e0900)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1510 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc057c73f80)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1330 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc059d92488, 0x1f5)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc00ab72ba0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d92460, 0xc059abf4d0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc057bdf830)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3e40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059d9c870}, 0x1, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc057bdf898, 0x3b9aca00, 0x0, 0xc0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc057bdf830, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc056197d60, 0xc000e0a7b8)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:447 +0x1caf

goroutine 1508 [select, 3 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc057c73f00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2850315 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc019f88000, 0xc01da3e5c0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1587 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 1504 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0580a7420, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc020551380}, 0xc0560efce8, 0xc05ba7eb40, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0580a7420, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc05a0860f0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059cc7000, {0x354b360, 0xc05a086050}, 0x1, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0580a7420, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2852728 [select]:
net/http.(*persistConn).writeLoop(0xc064956fc0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1468 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc059acad80)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1320 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0561abf10, 0x0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2850316 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0587bcdd0, 0xc03061e580)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1329 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc059ad0840)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:204 +0xa7
created by k8s.io/client-go/util/workqueue.newQueue
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:62 +0x1af

goroutine 1336 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc059ad0900)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:204 +0xa7
created by k8s.io/client-go/util/workqueue.newQueue
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:62 +0x1af

goroutine 1337 [sync.Cond.Wait, 3 minutes]:
sync.runtime_notifyListWait(0xc059d92668, 0x166)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc0357bb4c0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d92640, 0xc059abfda0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc059db67e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3e40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059d9cfc0}, 0x1, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059db6848, 0x3b9aca00, 0x0, 0x0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc059db67e0, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc056197f40, 0xc056213e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:447 +0x1caf

goroutine 2360 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06d2ed930, 0xc060121200)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2524 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0000ff110, 0x0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1316 [select, 5 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f52a80, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc01ee3be40}, 0xc059af5ce8, 0xc059fdb1a0, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f52a80, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059abb4a0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059a9bac0, {0x354b360, 0xc059abb400}, 0x1, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f52a80, 0xc059a8e8a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1479 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 1251 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0557110a0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0589c1c00)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 704340 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc021874750, 0xc05b19ccc0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2730499 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917f8e0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02d6ae780, 0xc036823000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02d6ae780, {0xc036823000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02d6ae780, {0xc036823000, 0xc06d2d8c80, 0xc055ee7f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc01afb55d8, {0xc036823000, 0xc0274cf380, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc01e552d80, {0xc036823000, 0xc03f94b380, 0xc055ee7d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc0377e2960)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc0377e2960, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc01e552d80)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 3142 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916e5c0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02895ed00, 0xc0248fd000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02895ed00, {0xc0248fd000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02895ed00, {0xc0248fd000, 0xc06d2d8b40, 0xc014758f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05d6cd1f8, {0xc0248fd000, 0xc055d53040, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0115545a0, {0xc0248fd000, 0xc020e20720, 0xc014758d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc027d9c900)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc027d9c900, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0115545a0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1444 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d80540, 0x0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2686292 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc03dfd36c0, 0xc00181f620)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2684344 [select]:
net/http.(*persistConn).writeLoop(0xc05caade60)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2515 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0001b75f0, 0xc05f4b3600)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1744046 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917efd0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05e346480, 0xc02ebd3000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05e346480, {0xc02ebd3000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05e346480, {0xc02ebd3000, 0xc06d2d8140, 0xc005bc2f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc052dd2790, {0xc02ebd3000, 0xc029402340, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc029eaf8c0, {0xc02ebd3000, 0xc04ddf3f20, 0xc005bc2d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc04e1955c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc04e1955c0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc029eaf8c0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2370198 [IO wait]:
internal/poll.runtime_pollWait(0x7fc3884958f0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05d09ca00, 0xc0429ec000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05d09ca00, {0xc0429ec000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05d09ca00, {0xc0429ec000, 0xc06d2d8f00, 0xc055d55f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc01a862a20, {0xc0429ec000, 0xc022d44ea0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc017668360, {0xc0429ec000, 0xc060db2900, 0xc055d55d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc0259255c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc0259255c0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc017668360)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2452 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06549d2b0, 0xc05e062e80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1390 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc055f530a0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc03766dec0}, 0xc04dcfbce8, 0xc05a2d5b00, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc055f530a0, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc059dbe140)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059dbc100, {0x354b360, 0xc059dbe0a0}, 0x1, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc055f530a0, 0xc059a8f6e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2553120 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b91256c8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc057afba80, 0xc01d191000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc057afba80, {0xc01d191000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc057afba80, {0xc01d191000, 0xc06d2d8c80, 0xc00c307f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000d641b8, {0xc01d191000, 0xc04838b380, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc01c2d0fc0, {0xc01d191000, 0xc0017c2a80, 0xc00c307d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc037cc6000)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc037cc6000, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc01c2d0fc0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1692 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 2505000 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020db6270, 0xc028cd0100)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1326 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc059aca500)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1331 [sync.Cond.Wait, 9 minutes]:
sync.runtime_notifyListWait(0xc059d92528, 0x59)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc014bd66e0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d92500, 0xc059abf7c0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc057bdfd40)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3e40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059d9cae0}, 0x1, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc057bdfda8, 0x3b9aca00, 0x0, 0x80, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc057bdfd40, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc056197cc0, 0xc057b2cc00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:448 +0x1d6f

goroutine 1328 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc059aca580)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:742 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1332 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc059d925c8, 0x913)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc01e13d3c0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059d925a0, 0xc059abfab0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc059db62d0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9bd3e40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059d9cd50}, 0x1, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059db6338, 0x3b9aca00, 0x0, 0x0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc059db62d0, 0xc059a8f3e0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc056197e00, 0xc05708cd80)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:449 +0x1e2f

goroutine 1333 [chan receive, 889 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc059acaa00, {0x35a0c90, 0xc059b9cb80}, 0xc059b4d920)
	/app/discovery/kubernetes/endpoints.go:150 +0x2e5
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc059acaa00})
	/app/discovery/kubernetes/kubernetes.go:580 +0x75
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:578 +0x8cf

goroutine 2365914 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc031d5d1e0, 0xc021c42e80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1743547 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0393a2270, 0xc037a0d9e0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 874 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0580d70e0)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:204 +0xa7
created by k8s.io/client-go/util/workqueue.newQueue
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:62 +0x1af

goroutine 3172 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916e6a8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0287f2780, 0xc050c32400, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0287f2780, {0xc050c32400, 0x81e, 0x81e})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0287f2780, {0xc050c32400, 0x819, 0xc025981160})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc063f6fb68, {0xc050c32400, 0xc050c32400, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc02ab52a20, {0xc050c32400, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc028880278, {0x3544be0, 0xc02ab52a20})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc028880000, {0x7fc0b9166040, 0xc0287d1180}, 0x81e)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc028880000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc028880000, {0xc02882d000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc02882a480, {0xc011998ac0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc02882a480}, {0xc011998ac0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc011998ac0, 0x9, 0xc001056420}, {0x35449e0, 0xc02882a480})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc011998a80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00d6faf98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc0259db380)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 875 [sync.Cond.Wait, 3 minutes]:
sync.runtime_notifyListWait(0xc059cce208, 0x167)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc036b77940)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059cce1e0, 0xc0582d2650)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0582b77a0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9c04820)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059cc85d0}, 0x1, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0582b7808, 0x3b9aca00, 0x0, 0x40, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc0582b77a0, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc059cce0a0, 0xc0586c7180)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:447 +0x1caf

goroutine 876 [sync.Cond.Wait, 9 minutes]:
sync.runtime_notifyListWait(0xc059cce2a8, 0x59)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc01cb07860)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059cce280, 0xc0582d2940)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0582b7cb0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9c04820)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059cc8840}, 0x1, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0582b7d18, 0x3b9aca00, 0x0, 0x0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc0582b7cb0, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc059cce000, 0xc0586c6e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:448 +0x1d6f

goroutine 2505603 [select]:
net/http.(*persistConn).writeLoop(0xc0116ea000)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1248 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc058c1f880, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0589ee280)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1249 [select, 1 minutes]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc055710e00, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0589ee080)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1273 [select, 1 minutes]:
github.com/prometheus/prometheus/scrape.(*Manager).reloader(0xc0003b8600)
	/app/scrape/manager.go:174 +0x145
created by github.com/prometheus/prometheus/scrape.(*Manager).Run
	/app/scrape/manager.go:148 +0x65

goroutine 1302 [select, 3 minutes]:
github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc00084dc20, {0x35a0c90, 0xc059b9d900}, 0xc059d27220, 0xc059d3eb40)
	/app/discovery/legacymanager/manager.go:208 +0xdf
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:203 +0x452

goroutine 2640460 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917f370, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05888fc00, 0xc01d6c2000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05888fc00, {0xc01d6c2000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05888fc00, {0xc01d6c2000, 0xc06d2d8f00, 0xc00d6f6f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc016bd0b18, {0xc01d6c2000, 0xc0066b2d00, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc001d6e120, {0xc01d6c2000, 0xc025c02840, 0xc00d6f6d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc01c7fe360)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc01c7fe360, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc001d6e120)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2448 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06549cdd0, 0xc05e063080)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1787235 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917cc90, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc030136100, 0xc031414000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc030136100, {0xc031414000, 0x3cd7, 0x3cd7})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc030136100, {0xc031414000, 0x3cd2, 0xc03078c200})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc001760010, {0xc031414000, 0xc031414000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc015648018, {0xc031414000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0304b8278, {0x3544be0, 0xc015648018})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0304b8000, {0x7fc0b9166040, 0xc0302fa0c0}, 0x3cd7)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0304b8000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0304b8000, {0xc02ac5b000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc0313ac6c0, {0xc020ff62e0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc0313ac6c0}, {0xc020ff62e0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc020ff62e0, 0x9, 0xc001595d10}, {0x35449e0, 0xc0313ac6c0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc020ff62a0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01475bf98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc02998c180)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2850743 [select]:
net/http.(*persistConn).writeLoop(0xc00a1db0e0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 704333 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc031d5c8f0, 0xc056968380)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2856032 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc017c14f48, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc017c14f38)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc017c14f30, {0xc01e209800, 0x200, 0x200})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc01e209800, 0x217c472, 0xc055177cf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc037397680)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc037397680)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc037397680, {0x2748340, 0xc01fcef860})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc01b64be60, {0xc0342b9c00, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc01fc1aaa0, 0xc04cb68b40, {0x357ac00, 0xc0207011c0})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc01fc92860)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc020701180)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2842735 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc013dc64c8, 0xa)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x86)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc013dc64b0, {0xc030558001, 0x1dff, 0x1dff})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x87}, {0xc030558001, 0xc030558000, 0xc022287cf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc03141a780)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc03141a780)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc03141a780, {0x2748340, 0xc01fdb06c0})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc06e82c150, {0xc0156f0000, 0x2000, 0x2600})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc013e3d5e0, 0xc052df44d0, {0x357ac00, 0xc020701880})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc014840a00)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc02a54f600)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2639175 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0302605b0, 0xc0287c4510)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1502 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0580dd0a0, 0x0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2356 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06d2ed450, 0xc05f509aa0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 812 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 1498 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0580a7340, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc020701180}, 0xc055f5bce8, 0xc05b3e55c0, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0580a7340, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0582b1f90)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059cc6f00, {0x354b360, 0xc0582b1ef0}, 0x1, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0580a7340, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1503 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1496 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0580dcfc0, 0x0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1509 [chan receive, 9 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9c04820)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc058153f38, {0x354b380, 0xc059cc8d20}, 0x1, 0xc0580c3980)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc057b6ea62, 0x3b9aca00, 0x0, 0xa0, 0xc058153f88)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc057c73f80)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1497 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 810 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 1358 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc059d80150, 0xc059b9cd00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1491 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 1490 [chan receive, 889 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0580dd030, 0x0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:628 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1507 [chan receive, 3 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:772 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9c04820)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0581f5738, {0x354b380, 0xc059cc8cf0}, 0x1, 0xc0580c3920)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc03a65dc90, 0x3b9aca00, 0x0, 0xa0, 0xc0581f5788)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc057c73f00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:771 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 1492 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0580a7260, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc035bbee40}, 0xc055d43ce8, 0xc05b3e4d80, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:468 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0580a7260, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:428 +0x5f6
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0582b1e50)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc059cc6dc0, {0x354b360, 0xc0582b1db0}, 0x1, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0580a7260, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x13d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 2552630 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc014886000, 0xc0192030c0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 878 [chan receive, 889 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc057c73e80, {0x35a0c90, 0xc059b9d900}, 0xc059d3eb40)
	/app/discovery/kubernetes/endpoints.go:150 +0x2e5
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc057c73e80})
	/app/discovery/kubernetes/kubernetes.go:580 +0x75
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:578 +0x8cf

goroutine 877 [sync.Cond.Wait, 3 minutes]:
sync.runtime_notifyListWait(0xc059cce348, 0x25e)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc0357bb440)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc059cce320, 0xc0582d2c30)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:525 +0x198
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc05a088240)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fc0b9c04820)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc059cc8ab0}, 0x1, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc05a0882a8, 0x3b9aca00, 0x0, 0x0, 0x7fc0b9c9c8e0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc05a088240, 0xc059db5e00)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc059cce140, 0xc0583b2c80)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:410 +0x449
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:449 +0x1e2f

goroutine 1250 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc055710fc0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0589ee480)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 2853773 [select, 1 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc01b6d8480, 0xc02fe32100)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0xc0411c66c0, 0xc0411c66c0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 2372288 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba93578, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05e136000, 0xc0260d4000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05e136000, {0xc0260d4000, 0x3cd7, 0x3cd7})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05e136000, {0xc0260d4000, 0x3cd2, 0xc0224a7b40})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc03605e510, {0xc0260d4000, 0xc0260d4000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc013b575f0, {0xc0260d4000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0224ea278, {0x3544be0, 0xc013b575f0})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0224ea000, {0x7fc0b9166040, 0xc022497840}, 0x3cd7)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0224ea000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0224ea000, {0xc02b561000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc0495d7020, {0xc062908040, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc0495d7020}, {0xc062908040, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc062908040, 0x9, 0xc001c3a480}, {0x35449e0, 0xc0495d7020})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc062908000)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc055eccf98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01bbbdc80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2640355 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916b5b0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc033626680, 0xc005412000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc033626680, {0xc005412000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc033626680, {0xc005412000, 0xc06d2d8b40, 0xc005bc3f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000c2d00, {0xc005412000, 0xc0581b8820, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc014cc3440, {0xc005412000, 0xc04c01bec0, 0xc005bc3d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc0311927e0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc0311927e0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc014cc3440)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1781545 [select]:
net/http.(*persistConn).writeLoop(0xc035fca7e0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2463 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc05f41e750, 0xc06d253600)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2850431 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc03dfd2270, 0xc05fae2180)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2638166 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0285075f0, 0xc03837ce00)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 695970 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba93e88, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc03fe7e480, 0xc0369bf000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc03fe7e480, {0xc0369bf000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc03fe7e480, {0xc0369bf000, 0xc06d2d8640, 0xc000e83f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc01a882dc0, {0xc0369bf000, 0xc029b94340, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc02ba84b40, {0xc0369bf000, 0xc001a11260, 0xc000e83d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc037881aa0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc037881aa0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc02ba84b40)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2484 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc05f41e270, 0xc05f509aa0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2853422 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916ed00, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05ed86300, 0xc00943f000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05ed86300, {0xc00943f000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05ed86300, {0xc00943f000, 0x4d9ca6, 0x4d8e460})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc009e67e30, {0xc00943f000, 0x0, 0x0})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).Read(0xc01afe8e10, {0xc00943f000, 0x1000, 0x1000})
	/usr/local/go/src/net/http/server.go:780 +0x16d
bufio.(*Reader).fill(0xc03999d500)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03999d500, 0x4)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*conn).serve(0xc01afe4e60, {0x35a0d38, 0xc000f2f740})
	/usr/local/go/src/net/http/server.go:1955 +0xc36
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3034 +0x4e8

goroutine 2483 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc05f41e680, 0xc060121200)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1272 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0586ec000, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0588f7b80)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1546 [select, 9 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x11a
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2e5

goroutine 2498 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc000553930, 0xc05f4e1b00)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2842682 [select, 5 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc013dc6180, 0xc00eb0d300)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a16a0, 0xc026e97780)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 1246 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0583c2b60, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0589ee680)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1247 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0583c36c0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0589ee980)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 2638165 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc028507520, 0xc0587ad800)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1254 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0557117a0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0589eef80)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1253 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc055710c40, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0589ef280)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 2855063 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc0091e8f48, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc0091e8f38)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc0091e8f30, {0xc012da4c00, 0x200, 0x200})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc012da4c00, 0xc055f0b560, 0xc055f0b4f0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc02dbba000)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc02dbba000)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc02dbba000, {0x2748340, 0xc01f138c60})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc02bb064b0, {0xc0366a8c00, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc01784ef00, 0xc059b9cd00, {0x357ac00, 0xc045d07b40})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc01609d740)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc045d07b00)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2554881 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba930f0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc026136a00, 0xc01da88000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc026136a00, {0xc01da88000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc026136a00, {0xc01da88000, 0xc06d2d8640, 0xc00ed16f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc02661b568, {0xc01da88000, 0xc0583b0680, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc005714b40, {0xc01da88000, 0xc03b63da40, 0xc00ed16d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc041472000)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc041472000, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc005714b40)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2841021 [sync.Cond.Wait, 5 minutes]:
sync.runtime_notifyListWait(0xc01c4a1248, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc021b7cba0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc01c4a1230, {0xc01f166000, 0x200, 0x200})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc01f166000, 0xc058151d60, 0xc058151cf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc02fafbe00)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc02fafbe00)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc02fafbe00, {0x2748340, 0xc01f138cd8})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc01eea5260, {0xc01f0ef000, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc01f142370, 0xc059b9cb00, {0x357ac00, 0xc01ee3be80})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc01f14e260)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc01ee3be40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2503 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0005532b0, 0xc05f4e0f00)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2856031 [select, 1 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc017c14f00, 0xc033d9ed00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc01b2f8480)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 1271 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc055711500, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0588f7d80)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1280 [select, 3 minutes]:
github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc00084dae0, {0x35a0c90, 0xc059b9cc00}, 0xc059a56280, 0xc059b4d980)
	/app/discovery/legacymanager/manager.go:208 +0xdf
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:203 +0x452

goroutine 1252 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0557115e0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0589ef680)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1279 [semacquire, 889 minutes]:
sync.runtime_Semacquire(0x0)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc0596bd540)
	/usr/local/go/src/sync/waitgroup.go:130 +0x71
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0596bd540, {0x35a0c90, 0xc059b9cc00}, 0xc059b4d980)
	/app/discovery/kubernetes/kubernetes.go:586 +0x746
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:202 +0x3b3

goroutine 1278 [select, 1 minutes]:
github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc00084dae0, {0x35a0c90, 0xc059b9cb80}, 0xc0596fb310, 0xc059b4d920)
	/app/discovery/legacymanager/manager.go:208 +0xdf
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:203 +0x452

goroutine 1297 [semacquire, 889 minutes]:
sync.runtime_Semacquire(0x0)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc0596bd680)
	/usr/local/go/src/sync/waitgroup.go:130 +0x71
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0596bd680, {0x35a0c90, 0xc059b9cc80}, 0xc059b4d9e0)
	/app/discovery/kubernetes/kubernetes.go:586 +0x746
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:202 +0x3b3

goroutine 1784044 [select]:
net/http.(*persistConn).writeLoop(0xc004754a20)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1275 [semacquire, 889 minutes]:
sync.runtime_Semacquire(0x0)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc0596bd2c0)
	/usr/local/go/src/sync/waitgroup.go:130 +0x71
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0596bd2c0, {0x35a0c90, 0xc059b9cb00}, 0xc059b4d860)
	/app/discovery/kubernetes/kubernetes.go:586 +0x746
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:202 +0x3b3

goroutine 1298 [select, 9 minutes]:
github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc00084dae0, {0x35a0c90, 0xc059b9cc80}, 0xc059a571d0, 0xc059b4d9e0)
	/app/discovery/legacymanager/manager.go:208 +0xdf
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:203 +0x452

goroutine 2714234 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917f540, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc012171c80, 0xc0195f6000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc012171c80, {0xc0195f6000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc012171c80, {0xc0195f6000, 0xc06d2d8b40, 0xc0402d4f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00cfac930, {0xc0195f6000, 0xc000eb7040, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc01a2db8c0, {0xc0195f6000, 0xc01fa63740, 0xc0402d4d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc03d9c1560)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03d9c1560, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc01a2db8c0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1299 [semacquire, 889 minutes]:
sync.runtime_Semacquire(0x0)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc0596bd7c0)
	/usr/local/go/src/sync/waitgroup.go:130 +0x71
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0596bd7c0, {0x35a0c90, 0xc059b9cd00}, 0xc059b4da40)
	/app/discovery/kubernetes/kubernetes.go:586 +0x746
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:202 +0x3b3

goroutine 1300 [select, 9 minutes]:
github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc00084dae0, {0x35a0c90, 0xc059b9cd00}, 0xc059d26140, 0xc059b4da40)
	/app/discovery/legacymanager/manager.go:208 +0xdf
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:203 +0x452

goroutine 1301 [semacquire, 889 minutes]:
sync.runtime_Semacquire(0x0)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc0596bda40)
	/usr/local/go/src/sync/waitgroup.go:130 +0x71
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0596bda40, {0x35a0c90, 0xc059b9d900}, 0xc059d3eb40)
	/app/discovery/kubernetes/kubernetes.go:586 +0x746
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:202 +0x3b3

goroutine 1270 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc05050c8c0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc0588f7f80)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1255 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0583c3960, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc058c89280)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1268 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0557116c0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896e580)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1267 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc058c1f180, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896e780)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1266 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc058c1e000, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896e980)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1265 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc058c1e0e0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896eb80)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1264 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0586ecc40, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896ef00)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1263 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc05633e2a0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896f100)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1269 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0586eddc0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896f300)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1261 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc055711c00, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896f500)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1260 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc055710d20, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896f880)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1259 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc05050d5e0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896fa80)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1262 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0586edce0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896fc80)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1257 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0583c3340, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc05896fe80)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1258 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc05050d420, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc058992180)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1256 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc055711ce0, {0x35a0cc8, 0xc00005a018})
	/app/rules/manager.go:415 +0x8b5
github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc058992580)
	/app/rules/manager.go:999 +0xaf
created by github.com/prometheus/prometheus/rules.(*Manager).Update
	/app/rules/manager.go:989 +0x3b2

goroutine 1822 [sync.Cond.Wait, 9 minutes]:
sync.runtime_notifyListWait(0xc057d77e90, 0xb0)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x24e9980)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/util/workqueue.(*Type).Get(0xc059ad09c0)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:151 +0x9e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc059acaf00, {0x35a0c90, 0xc059b9cc80}, 0x82b825)
	/app/discovery/kubernetes/endpoints.go:154 +0x6e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1()
	/app/discovery/kubernetes/endpoints.go:145 +0x53
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:144 +0x2c5

goroutine 1277 [semacquire, 889 minutes]:
sync.runtime_Semacquire(0x0)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc0596bd400)
	/usr/local/go/src/sync/waitgroup.go:130 +0x71
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0596bd400, {0x35a0c90, 0xc059b9cb80}, 0xc059b4d920)
	/app/discovery/kubernetes/kubernetes.go:586 +0x746
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:202 +0x3b3

goroutine 1276 [select, 2 minutes]:
github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc00084dae0, {0x35a0c90, 0xc059b9cb00}, 0xc0596fa3c0, 0xc059b4d860)
	/app/discovery/legacymanager/manager.go:208 +0xdf
created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider
	/app/discovery/legacymanager/manager.go:203 +0x452

goroutine 2714237 [select]:
net/http.(*persistConn).writeLoop(0xc028138ea0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2439 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06549c9c0, 0xc05e48a800)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2369024 [select]:
net/http.(*persistConn).writeLoop(0xc06a4ba480)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2433 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06549cd00, 0xc05e48b880)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1784012 [select]:
net/http.(*persistConn).writeLoop(0xc02a162a20)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2848973 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc024ef33c8, 0x2)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xa7)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc024ef33b0, {0xc00fb1a001, 0x5ff, 0x5ff})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0xa8}, {0xc00fb1a001, 0xc00fb1a000, 0xc022289cf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc032fa2b40)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc032fa2b40)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc032fa2b40, {0x2748340, 0xc01d31c9d8})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc03319a360, {0xc032729c00, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0331765a0, 0xc028eae360, {0x357ac00, 0xc0178d5e80})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0331964a0)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc03319c0c0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2504717 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc016fa52b0, 0xc01a313260)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1787557 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9054d28, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02bad4a00, 0xc000134400, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02bad4a00, {0xc000134400, 0x81e, 0x81e})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02bad4a00, {0xc000134400, 0x819, 0xc053f0a300})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc002282b70, {0xc000134400, 0xc000134400, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0174a1998, {0xc000134400, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0567a1778, {0x3544be0, 0xc0174a1998})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0567a1500, {0x7fc0b9166040, 0xc05d2c1b80}, 0x81e)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0567a1500, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0567a1500, {0xc0587a1000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc01048e600, {0xc05129d2a0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc01048e600}, {0xc05129d2a0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc05129d2a0, 0x9, 0xc001bcdaa0}, {0x35449e0, 0xc01048e600})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc05129d260)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc059006f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01ebca480)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2966 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916aad0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc020a02580, 0xc021b76000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc020a02580, {0xc021b76000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc020a02580, {0xc021b76000, 0xc06d2d8c80, 0xc021b7ef68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc063f6e848, {0xc021b76000, 0xc057f4d040, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc00f0afc20, {0xc021b76000, 0xc001c5bb00, 0xc021b7ed30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc021b6c6c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc021b6c6c0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc00f0afc20)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1694 [sync.Cond.Wait, 2 minutes]:
sync.runtime_notifyListWait(0xc057d77790, 0x304)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x24e9980)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/util/workqueue.(*Type).Get(0xc059ad0240)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:151 +0x9e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc059aca480, {0x35a0c90, 0xc059b9cb00}, 0x82b825)
	/app/discovery/kubernetes/endpoints.go:154 +0x6e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1()
	/app/discovery/kubernetes/endpoints.go:145 +0x53
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:144 +0x2c5

goroutine 1743768 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc000ec9ee0, 0x4dbf460)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2967 [select]:
net/http.(*persistConn).writeLoop(0xc00f0afc20)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2506948 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9054fe0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc057b98f80, 0xc010f20000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc057b98f80, {0xc010f20000, 0x3c7d, 0x3c7d})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc057b98f80, {0xc010f20000, 0x2c59, 0xc046a23200})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0257b5770, {0xc010f20000, 0xc010f2101f, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc016232930, {0xc010f20000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0061eccf8, {0x3544be0, 0xc016232930})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0061eca80, {0x7fc0b9166040, 0xc02f5e75c0}, 0x2c5e)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0061eca80, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0061eca80, {0xc00fb01000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc02aeddbc0, {0xc0262d6ac0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc02aeddbc0}, {0xc0262d6ac0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0262d6ac0, 0x9, 0xc001693200}, {0x35449e0, 0xc02aeddbc0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0262d6a80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc059012f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc035452900)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 1820 [sync.Cond.Wait, 3 minutes]:
sync.runtime_notifyListWait(0xc057d77dd0, 0x1c4)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x24e9980)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/util/workqueue.(*Type).Get(0xc059ad0900)
	/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:151 +0x9e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc059acac80, {0x35a0c90, 0xc059b9cc00}, 0x82b825)
	/app/discovery/kubernetes/endpoints.go:154 +0x6e
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1()
	/app/discovery/kubernetes/endpoints.go:145 +0x53
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:144 +0x2c5

goroutine 2238 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06549dc70, 0xc06abaf700)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2732953 [select]:
net/http.(*persistConn).writeLoop(0xc02a88fd40)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2684221 [select]:
net/http.(*persistConn).writeLoop(0xc032adcc60)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2480 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc000553a00, 0xc05f2c3600)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2418 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06d2ec820, 0xc06d221400)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2539 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc000108f70, 0xc05e062e80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2541 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc000109520, 0xc06abaf700)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 699885 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9187a50, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05ed83a00, 0xc018998000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05ed83a00, {0xc018998000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05ed83a00, {0xc018998000, 0xc06d2d8c80, 0xc05900bf68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc029b43c90, {0xc018998000, 0xc055eb5520, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc00c8fa120, {0xc018998000, 0xc02d9f3aa0, 0xc05900bd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc04db8df20)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc04db8df20, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc00c8fa120)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2710680 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc017594f70, 0xc024111830)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2735101 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916af58, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0567fcb80, 0xc0478e6400, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0567fcb80, {0xc0478e6400, 0x81e, 0x81e})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0567fcb80, {0xc0478e6400, 0x7de, 0xc042118380})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc02661a108, {0xc0478e6400, 0xc0478e643b, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc015001ce0, {0xc0478e6400, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc00c934278, {0x3544be0, 0xc015001ce0})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc00c934000, {0x7fc0b9166040, 0xc06ca2a480}, 0x7e3)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc00c934000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc00c934000, {0xc016a66000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc043cd2420, {0xc00c946120, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc043cd2420}, {0xc00c946120, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00c946120, 0x9, 0xc001598d20}, {0x35449e0, 0xc043cd2420})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00c9460e0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc036caff98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00c834780)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2640356 [select]:
net/http.(*persistConn).writeLoop(0xc014cc3440)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2852727 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9180b60, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc036f75d80, 0xc036c0d000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc036f75d80, {0xc036c0d000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc036f75d80, {0xc036c0d000, 0xc06d2d8640, 0xc00e952f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05615cff8, {0xc036c0d000, 0xc0066b3a00, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc064956fc0, {0xc036c0d000, 0xc035ef6b40, 0xc00e952d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc036bab020)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc036bab020, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc064956fc0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2549 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc000109450, 0xc05f2ebc80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2567 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0005f6410, 0xc00084dae0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2507282 [select]:
net/http.(*persistConn).writeLoop(0xc01deadb00)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 695263 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9125c38, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc03f55dd00, 0xc06546c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc03f55dd00, {0xc06546c000, 0x3cd7, 0x3cd7})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc03f55dd00, {0xc06546c000, 0x3cd2, 0xc04bff9bc0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05cdba830, {0xc06546c000, 0xc06546c000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc025c821c8, {0xc06546c000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0214cf3f8, {0x3544be0, 0xc025c821c8})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0214cf180, {0x7fc0b9166040, 0xc03adeb140}, 0x3cd7)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0214cf180, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0214cf180, {0xc03e090000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc05a0c86c0, {0xc021887540, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc05a0c86c0}, {0xc021887540, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc021887540, 0x9, 0xc001aaac90}, {0x35449e0, 0xc05a0c86c0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc021887500)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc05900cf98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc0218c4000)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2851851 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9180250, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc01a1b6100, 0xc01a1ec000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc01a1b6100, {0xc01a1ec000, 0x3cd5, 0x3cd5})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc01a1b6100, {0xc01a1ec000, 0x3cd0, 0xc01cc2c3a0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc009d87d80, {0xc01a1ec000, 0xc01a1ec000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc00051ad38, {0xc01a1ec000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc002bd8978, {0x3544be0, 0xc00051ad38})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc002bd8700, {0x7fc0b9166040, 0xc01cc1c4c0}, 0x3cd5)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc002bd8700, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc002bd8700, {0xc00eed7000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc0317cdda0, {0xc01cc0a580, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc0317cdda0}, {0xc01cc0a580, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc01cc0a580, 0x9, 0xc001830120}, {0x35449e0, 0xc0317cdda0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc01cc0a540)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00e953f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01c9a0900)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2845874 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc02da83cc8, 0x2a)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xa7)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc02da83cb0, {0xc025300001, 0x3dff, 0x3dff})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0xa8}, {0xc025300001, 0xc025300000, 0xc0560f3cf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc024086a00)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc024086a00)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc024086a00, {0x2748340, 0xc01db6d188})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc025b933e0, {0xc00e2a8000, 0x4000, 0x4a80})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0102581e0, 0xc047b482d0, {0x357ac00, 0xc03f04ee80})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc033486c60)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc013826e40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2372144 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc021d55a00, 0xc056c814c0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2831437 [select, 8 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc01d001b00, 0xc020a51a00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a0d38, 0xc000f2f740)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 2633 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388495638, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05d163b00, 0xc0140a2000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05d163b00, {0xc0140a2000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05d163b00, {0xc0140a2000, 0xc06d2d83c0, 0xc00e4a1f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc06d262868, {0xc0140a2000, 0xc055172820, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc00587cd80, {0xc0140a2000, 0xc0013a3c20, 0xc00e4a1d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc00680fa40)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc00680fa40, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc00587cd80)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2634 [select]:
net/http.(*persistConn).writeLoop(0xc00587cd80)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2637 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388495550, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc008a62400, 0xc00bf8a000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc008a62400, {0xc00bf8a000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc008a62400, {0xc00bf8a000, 0xc06d2d9400, 0xc055ed1f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc06d263310, {0xc00bf8a000, 0xc0562f3040, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0045ac7e0, {0xc00bf8a000, 0xc0013aaa80, 0xc055ed1d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc008a310e0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc008a310e0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0045ac7e0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2638 [select]:
net/http.(*persistConn).writeLoop(0xc0045ac7e0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2507785 [select]:
net/http.(*persistConn).writeLoop(0xc02808fb00)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2638901 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9180fe8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05812bf80, 0xc0129bb000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05812bf80, {0xc0129bb000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05812bf80, {0xc0129bb000, 0xc06d2d8c80, 0xc00b95bf68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0340631d0, {0xc0129bb000, 0xc00a266b60, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc03092eb40, {0xc0129bb000, 0xc0318ef6e0, 0xc00b95bd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc066bac3c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc066bac3c0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc03092eb40)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1744583 [select]:
net/http.(*persistConn).writeLoop(0xc020d60360)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1744582 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917ce60, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc021128980, 0xc01bcfe000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc021128980, {0xc01bcfe000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc021128980, {0xc01bcfe000, 0x4486af, 0xc007447ec8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc01392f5c8, {0xc01bcfe000, 0x0, 0x2})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc020d60360, {0xc01bcfe000, 0x448c40, 0xc007447ec8})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc057891740)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc057891740, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc020d60360)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2554882 [select]:
net/http.(*persistConn).writeLoop(0xc005714b40)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2851047 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e2500d0, 0xc0307b60c0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 3025 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916abb8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc01e606180, 0xc00239a000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc01e606180, {0xc00239a000, 0x3c7b, 0x3c7b})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc01e606180, {0xc00239a000, 0x3c76, 0x413b05})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05f223858, {0xc00239a000, 0x7fc3883c56e0, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc00051a5e8, {0xc00239a000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc01f0a6278, {0x3544be0, 0xc00051a5e8})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc01f0a6000, {0x7fc0b9166040, 0xc01e1bb900}, 0x3c7b)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc01f0a6000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc01f0a6000, {0xc056079000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc01f0ab500, {0xc00eef4040, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc01f0ab500}, {0xc00eef4040, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00eef4040, 0x9, 0xc00191def0}, {0x35449e0, 0xc01f0ab500})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00eef4000)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00744af98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01e1bf380)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2735608 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9180508, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05efce480, 0xc00d960000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05efce480, {0xc00d960000, 0x3cd7, 0x3cd7})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05efce480, {0xc00d960000, 0x3cd2, 0xc022c832a0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc02349aad8, {0xc00d960000, 0xc00d960000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc01738c9f0, {0xc00d960000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc00d5885f8, {0x3544be0, 0xc01738c9f0})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc00d588380, {0x7fc0b9166040, 0xc01624b700}, 0x3cd7)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc00d588380, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc00d588380, {0xc0055fd000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc029d3f200, {0xc00df86ba0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc029d3f200}, {0xc00df86ba0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00df86ba0, 0x9, 0xc0015ee180}, {0x35449e0, 0xc029d3f200})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00df86b60)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00e4a2f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00ca9cf00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 3035 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916f0a0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc021c7db00, 0xc021ca3000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc021c7db00, {0xc021ca3000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc021c7db00, {0xc021ca3000, 0xc06d2d8c80, 0xc002be1f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05f223ce8, {0xc021ca3000, 0xc057574b60, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc01e597b00, {0xc021ca3000, 0xc01b0e81e0, 0xc002be1d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc021c4c6c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc021c4c6c0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc01e597b00)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1784392 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917f7f8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00eb8bc80, 0xc03e091000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00eb8bc80, {0xc03e091000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00eb8bc80, {0xc03e091000, 0xc06d2d8b40, 0xc055d6df68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0144bab10, {0xc03e091000, 0xc03d883040, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0027939e0, {0xc03e091000, 0xc025dfee40, 0xc055d6dd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc03c4e2a80)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03c4e2a80, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0027939e0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 3036 [select]:
net/http.(*persistConn).writeLoop(0xc01e597b00)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2552501 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc00c324d00, 0xc014e70580)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2369023 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917f288, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05f2a7800, 0xc0355ef000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05f2a7800, {0xc0355ef000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05f2a7800, {0xc0355ef000, 0xc06d2d8b40, 0xc010135f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc001c0a340, {0xc0355ef000, 0xc04c708ea0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc06a4ba480, {0xc0355ef000, 0xc001eaeae0, 0xc010135d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc04b70ae40)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc04b70ae40, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc06a4ba480)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 3649 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba93748, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc03d7b7580, 0xc03d886000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc03d7b7580, {0xc03d886000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc03d7b7580, {0xc03d886000, 0x4d9ca6, 0x4d8e460})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc005f533d0, {0xc03d886000, 0x0, 0x0})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).Read(0xc03d876420, {0xc03d886000, 0x1000, 0x1000})
	/usr/local/go/src/net/http/server.go:780 +0x16d
bufio.(*Reader).fill(0xc03d880d20)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03d880d20, 0x4)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*conn).serve(0xc03d86d5e0, {0x35a0d38, 0xc000f2f740})
	/usr/local/go/src/net/http/server.go:1955 +0xc36
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3034 +0x4e8

goroutine 704301 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06d268d00, 0xc056968180)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2638902 [select]:
net/http.(*persistConn).writeLoop(0xc03092eb40)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1821066 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9187228, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc03510ab80, 0xc03a5bd000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc03510ab80, {0xc03a5bd000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc03510ab80, {0xc03a5bd000, 0xc06d2d9540, 0xc00ed18f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc005908070, {0xc03a5bd000, 0xc04838b1e0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc02fea8c60, {0xc03a5bd000, 0xc032736e40, 0xc00ed18d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc05928bc20)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc05928bc20, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc02fea8c60)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2369420 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9180f00, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc04bc16d00, 0xc0270ce000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc04bc16d00, {0xc0270ce000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc04bc16d00, {0xc0270ce000, 0xc06d2d8640, 0xc00b958f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc004baf8b8, {0xc0270ce000, 0xc04e47fa00, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc03b1e4000, {0xc0270ce000, 0xc073e3f4a0, 0xc00b958d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc05852a0c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc05852a0c0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc03b1e4000)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2506626 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc014f42750, 0xc04b57bb40)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 3143 [select]:
net/http.(*persistConn).writeLoop(0xc0115545a0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1743996 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc024471110, 0xc0199f5d80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 3631 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba93830, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc03d7b7500, 0xc03d884000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc03d7b7500, {0xc03d884000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc03d7b7500, {0xc03d884000, 0xc06d2d88c0, 0xc005bc1f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc005f533c8, {0xc03d884000, 0xc058aaa000, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc01175c120, {0xc03d884000, 0xc0148942a0, 0xc005bc1d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc03d880c60)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03d880c60, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc01175c120)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2684127 [select]:
net/http.(*persistConn).writeLoop(0xc026593e60)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2372146 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc021d55ee0, 0xc01817d380)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2638167 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0285076c0, 0xc02b4d1f00)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2729222 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06d2ecb60, 0xc00180f740)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2733845 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba93da0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05d6ca000, 0xc00f6e2000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05d6ca000, {0xc00f6e2000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05d6ca000, {0xc00f6e2000, 0xc06d2d8f00, 0xc036cadf68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc03260caa8, {0xc00f6e2000, 0xc029b94b60, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc05cccb9e0, {0xc00f6e2000, 0xc01b894180, 0xc036cadd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc0599c0060)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc0599c0060, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc05cccb9e0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 3632 [select]:
net/http.(*persistConn).writeLoop(0xc01175c120)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2851785 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc02266de48, 0x1)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xa7)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc02266de30, {0xc052ec2c01, 0x5ff, 0x5ff})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0xa7}, {0xc052ec2c01, 0xc022ccf400, 0xc000e1dcf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc0326ce000)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc0326ce000)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc0326ce000, {0x2748340, 0xc025220030})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc022d10e70, {0xc022c3ac00, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc022d163c0, 0xc012e00d80, {0x357ac00, 0xc037fe5dc0})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0205454e0)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc020551380)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 694904 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc04287bd40, 0xc05dfef700)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2683569 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0563231e0, 0xc026d34400)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2858170 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba932c0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0314d0600, 0xc0314d8671, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0314d0600, {0xc0314d8671, 0x1, 0x1})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0314d0600, {0xc0314d8671, 0x3, 0x28a2620})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc01010af58, {0xc0314d8671, 0x405780, 0xc000e07720})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).backgroundRead(0xc0314d8660)
	/usr/local/go/src/net/http/server.go:672 +0x3f
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:668 +0xcf

goroutine 2841020 [select, 5 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc01c4a1200, 0xc05f5edf00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc01e26eff0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 2842440 [select, 5 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc00b08f800, 0xc058702500)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc02618fb90)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 2732058 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc000ec8410, 0xc020e06480)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2505602 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9055550, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02895fb80, 0xc00b135000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02895fb80, {0xc00b135000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02895fb80, {0xc00b135000, 0xc06d2d8c80, 0xc003080f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc03b7faf48, {0xc00b135000, 0xc05614cb60, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0116ea000, {0xc00b135000, 0xc0604ab3e0, 0xc003080d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc01e3c17a0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc01e3c17a0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0116ea000)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1779911 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc025582ea0, 0xc057850100)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2735153 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917d2e8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc058abf300, 0xc00cb12000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc058abf300, {0xc00cb12000, 0x3cd7, 0x3cd7})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc058abf300, {0xc00cb12000, 0x3cd2, 0xc01633a780})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00dc1a280, {0xc00cb12000, 0xc00cb12000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc014f78918, {0xc00cb12000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc008c47e78, {0x3544be0, 0xc014f78918})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc008c47c00, {0x7fc0b9166040, 0xc023804e80}, 0x3cd7)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc008c47c00, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc008c47c00, {0xc00373a000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc0666062a0, {0xc0568f5c40, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc0666062a0}, {0xc0568f5c40, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0568f5c40, 0x9, 0xc0010df020}, {0x35449e0, 0xc0666062a0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0568f5c00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc055ee4f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00ca47c80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 4395 [select]:
net/http.(*persistConn).writeLoop(0xc0244425a0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2681795 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01df0d930, 0xc0347eee10)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 703705 [select]:
net/http.(*persistConn).writeLoop(0xc050f23440)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2686260 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc05714e8f0, 0xc003016400)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 3693 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9054ef8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc03db1cf80, 0xc03db35000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc03db1cf80, {0xc03db35000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc03db1cf80, {0xc03db35000, 0xc06d2d8f00, 0xc00ed12f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0095323a0, {0xc03db35000, 0xc05861c680, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0127d18c0, {0xc03db35000, 0xc022ef7860, 0xc00ed12d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc03db30cc0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03db30cc0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0127d18c0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2715714 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917fd68, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc030a98e00, 0xc00f422000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc030a98e00, {0xc00f422000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc030a98e00, {0xc00f422000, 0xc06d2d8f00, 0xc000e86f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0583dfa70, {0xc00f422000, 0xc055f504e0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc02136d560, {0xc00f422000, 0xc05f508840, 0xc000e86d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc02e176840)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc02e176840, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc02136d560)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2733690 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9180338, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc058f89880, 0xc00bf8b000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc058f89880, {0xc00bf8b000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc058f89880, {0xc00bf8b000, 0xc06d2d8640, 0xc007acdf68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc02661a128, {0xc00bf8b000, 0xc000e10d00, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc06a763200, {0xc00bf8b000, 0xc001ed6c00, 0xc007acdd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc038895500)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc038895500, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc06a763200)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 4222 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9125d20, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00ceba080, 0xc00e4cc000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00ceba080, {0xc00e4cc000, 0x3daa, 0x3daa})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00ceba080, {0xc00e4cc000, 0x3da5, 0xc00ce9c8a0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc06d2633f8, {0xc00e4cc000, 0xc00e4cc000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc01651c3d8, {0xc00e4cc000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc055c5ecf8, {0x3544be0, 0xc01651c3d8})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc055c5ea80, {0x7fc0b9166040, 0xc02d707c80}, 0x3daa)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc055c5ea80, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc055c5ea80, {0xc00cedf000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc008a301e0, {0xc00cd34c80, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc008a301e0}, {0xc00cd34c80, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00cd34c80, 0x9, 0xc0016035f0}, {0x35449e0, 0xc008a301e0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00cd34c40)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc06e005f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00473ca80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2841064 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc01b79de48, 0x1d)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xa7)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc01b79de30, {0xc0027ba001, 0x3dff, 0x3dff})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0xa8}, {0xc0027ba001, 0xc0027ba000, 0xc000e1bcf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc01eee1cc0)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc01eee1cc0)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc01eee1cc0, {0x2748340, 0xc016bf86c0})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc01d865f50, {0xc00e2aca80, 0x4000, 0x4a80})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc01d8352c0, 0xc0161da120, {0x357ac00, 0xc022f35f00})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc01e38a1e0)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc01df44c40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 4139 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917c9d8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00bd18080, 0xc00bddd000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00bd18080, {0xc00bddd000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00bd18080, {0xc00bddd000, 0xc06d2d8640, 0xc005bc7f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc063f6e280, {0xc00bddd000, 0xc000eb69c0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc01ea27320, {0xc00bddd000, 0xc04387fd40, 0xc005bc7d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc057846180)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc057846180, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc01ea27320)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2367661 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917fab0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02c67a780, 0xc06ba6a000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02c67a780, {0xc06ba6a000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02c67a780, {0xc06ba6a000, 0xc06d2d8c80, 0xc05ac78f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc026588158, {0xc06ba6a000, 0xc05614d520, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc000f79200, {0xc06ba6a000, 0xc0201c8c60, 0xc05ac78d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc03d393380)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03d393380, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc000f79200)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2715315 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc028892a90, 0xc02f7fafc0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 3686 [select]:
net/http.(*persistConn).writeLoop(0xc0127d1320)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 4122 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b91260c0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00be99300, 0xc00946f000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00be99300, {0xc00946f000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00be99300, {0xc00946f000, 0xc06d2d8f00, 0xc00ed15f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00af9df20, {0xc00946f000, 0xc05851b1e0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc028d4a120, {0xc00946f000, 0xc0405bd920, 0xc00ed15d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc060136720)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc060136720, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc028d4a120)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2735023 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc059d07380, 0xc00abf4b80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 4236 [select]:
net/http.(*persistConn).writeLoop(0xc014fbb440)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 3685 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba933a8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc03db1cc00, 0xc03db33000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc03db1cc00, {0xc03db33000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc03db1cc00, {0xc03db33000, 0xc06d2d8a00, 0xc003390f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc009532310, {0xc03db33000, 0xc04f6e3520, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0127d1320, {0xc03db33000, 0xc022ef7140, 0xc003390d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc03db30420)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03db30420, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0127d1320)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 3635 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0bba93918, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc03ca76380, 0xc05106ed00, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc03ca76380, {0xc05106ed00, 0x81c, 0x81c})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc03ca76380, {0xc05106ed00, 0x7dc, 0xc03c891280})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00791a300, {0xc05106ed00, 0xc05106ed3b, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc014e91740, {0xc05106ed00, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0377dfe78, {0x3544be0, 0xc014e91740})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0377dfc00, {0x7fc0b9166040, 0xc03ca00ec0}, 0x7e1)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0377dfc00, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0377dfc00, {0xc03d60e000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc03d60c300, {0xc014a39a80, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc03d60c300}, {0xc014a39a80, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc014a39a80, 0x9, 0xc0015139e0}, {0x35449e0, 0xc03d60c300})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc014a39a40)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc000db2f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc03c824480)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 4140 [select]:
net/http.(*persistConn).writeLoop(0xc01ea27320)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2506199 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b91805f0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc027415000, 0xc027922000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc027415000, {0xc027922000, 0x3cd9, 0x3cd9})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc027415000, {0xc027922000, 0x3cd4, 0xc026c7c6a0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc032fe3fc0, {0xc027922000, 0xc027922000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc025c82528, {0xc027922000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc013755778, {0x3544be0, 0xc025c82528})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc013755500, {0x7fc0b9166040, 0xc026c60c80}, 0x3cd9)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc013755500, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc013755500, {0xc0271f1000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc01aa4e300, {0xc02503d000, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc01aa4e300}, {0xc02503d000, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc02503d000, 0x9, 0xc001aab050}, {0x35449e0, 0xc01aa4e300})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc02503cfc0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00744df98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc026c1e600)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 694905 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc04287bee0, 0xc063712080)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 699886 [select]:
net/http.(*persistConn).writeLoop(0xc00c8fa120)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 4338 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9125410, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00d1bc600, 0xc01a216000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00d1bc600, {0xc01a216000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00d1bc600, {0xc01a216000, 0xc06d2d8b40, 0xc00b959f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05d6cca18, {0xc01a216000, 0xc0562f2b60, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc00d1e46c0, {0xc01a216000, 0xc0188fe420, 0xc00b959d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc02e177860)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc02e177860, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc00d1e46c0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1782333 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc021d54750, 0xc064b13170)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1743548 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0393a2340, 0xc01a6d49c0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1821067 [select]:
net/http.(*persistConn).writeLoop(0xc02fea8c60)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2683530 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01ecf4000, 0xc039ec9980)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 3999 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917cd78, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00472a380, 0xc0032b1000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00472a380, {0xc0032b1000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00472a380, {0xc0032b1000, 0x4486af, 0xc055f23ec8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00af9dcb0, {0xc0032b1000, 0x0, 0x2})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc006800240, {0xc0032b1000, 0x448c40, 0xc055f23ec8})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc05f85b080)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc05f85b080, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc006800240)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2686950 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9125240, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc012645d00, 0xc01f1b0000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc012645d00, {0xc01f1b0000, 0x3cd7, 0x3cd7})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc012645d00, {0xc01f1b0000, 0x3cd2, 0xc017d32f80})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0359e2f38, {0xc01f1b0000, 0xc01f1b0000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc017721f68, {0xc01f1b0000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0169cacf8, {0x3544be0, 0xc017721f68})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0169caa80, {0x7fc0b9166040, 0xc01791e280}, 0x3cd7)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0169caa80, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0169caa80, {0xc017d7b000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc055239020, {0xc01a756820, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc055239020}, {0xc01a756820, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc01a756820, 0x9, 0xc001824a20}, {0x35449e0, 0xc055239020})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc01a7567e0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00ed14f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc017d74000)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2852421 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9125e08, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0332fb400, 0xc0339ec000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0332fb400, {0xc0339ec000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0332fb400, {0xc0339ec000, 0xc06d2d8b40, 0xc00ed13f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc026588210, {0xc0339ec000, 0xc012271860, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc05f585e60, {0xc0339ec000, 0xc0272a3bc0, 0xc00ed13d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc05619f920)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc05619f920, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc05f585e60)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 3951 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917d118, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc003292380, 0xc0032ae000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc003292380, {0xc0032ae000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc003292380, {0xc0032ae000, 0xc06d2d8640, 0xc00e3d7f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00af9c618, {0xc0032ae000, 0xc04026cb60, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc00e6da900, {0xc0032ae000, 0xc025d3daa0, 0xc00e3d7d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc06d1fce40)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc06d1fce40, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc00e6da900)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 4538 [select]:
net/http.(*persistConn).writeLoop(0xc035362240)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 3952 [select]:
net/http.(*persistConn).writeLoop(0xc00e6da900)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 4450 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917f0b8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc026f8be00, 0xc02c216000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc026f8be00, {0xc02c216000, 0x3cd5, 0x3cd5})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc026f8be00, {0xc02c216000, 0x3cd0, 0xc026fc88a0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00b4c2a00, {0xc02c216000, 0xc02c216000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0174bfc80, {0xc02c216000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc007678cf8, {0x3544be0, 0xc0174bfc80})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc007678a80, {0x7fc0b9166040, 0xc0270c4700}, 0x3cd5)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc007678a80, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc007678a80, {0xc028b5e000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc028b4d500, {0xc020ff6900, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc028b4d500}, {0xc020ff6900, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc020ff6900, 0x9, 0xc001bcdf20}, {0x35449e0, 0xc028b4d500})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc020ff68c0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00338df98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc0270c2d80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2845873 [select, 3 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc02da83c80, 0xc03887df00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc03b8f8b40)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 2369421 [select]:
net/http.(*persistConn).writeLoop(0xc03b1e4000)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 4125 [select]:
net/http.(*persistConn).writeLoop(0xc028d4a240)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2856357 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc03abfd248, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc00e3d5ba0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc03abfd230, {0xc02a200800, 0x200, 0x200})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc02a200800, 0x217c472, 0xc0551754f0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc028bcc780)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc028bcc780)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc028bcc780, {0x2748340, 0xc05dd7ddb8})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc01d253710, {0xc02c3aa400, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc000d56d70, 0xc055175638, {0x357ac00, 0xc03766df00})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc03855e840)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc03766dec0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 4124 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9125fd8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00be99400, 0xc00cc32000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00be99400, {0xc00cc32000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00be99400, {0xc00cc32000, 0xc06d2d8c80, 0xc055fb9f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00af9df30, {0xc00cc32000, 0xc057575520, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc028d4a240, {0xc00cc32000, 0xc04bae0000, 0xc055fb9d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc05f3bdc80)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc05f3bdc80, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc028d4a240)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 4123 [select]:
net/http.(*persistConn).writeLoop(0xc028d4a120)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1784043 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388495ba8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc033627580, 0xc00088c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc033627580, {0xc00088c000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc033627580, {0xc00088c000, 0xc06d2d8640, 0xc00c305f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc025ba7be0, {0xc00088c000, 0xc00a266680, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc004754a20, {0xc00088c000, 0xc001c73080, 0xc00c305d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc055fd9860)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc055fd9860, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc004754a20)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1745840 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388495298, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0399f7100, 0xc0051fb000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0399f7100, {0xc0051fb000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0399f7100, {0xc0051fb000, 0xc06d2d8a00, 0xc010138f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000f6cd10, {0xc0051fb000, 0xc04838ad00, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc01fdc4d80, {0xc0051fb000, 0xc033233260, 0xc010138d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc03d9c1380)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03d9c1380, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc01fdc4d80)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2715167 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc02299d110, 0xc01c0e9900)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 3961 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917d030, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0046ddf00, 0xc00456a000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0046ddf00, {0xc00456a000, 0x3e9e, 0x3e9e})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0046ddf00, {0xc00456a000, 0x3e99, 0xc03be2a940})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05f2221e0, {0xc00456a000, 0xc00456a000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc015648390, {0xc00456a000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0008dbaf8, {0x3544be0, 0xc015648390})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0008db880, {0x7fc0b9166040, 0xc0185f59c0}, 0x3e9e)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0008db880, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0008db880, {0xc00472c000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc0589f5200, {0xc06ccf0d60, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc0589f5200}, {0xc06ccf0d60, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc06ccf0d60, 0x9, 0xc001595e90}, {0x35449e0, 0xc0589f5200})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc06ccf0d20)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc003085f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01ec29c80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2850430 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020db7ba0, 0xc026b8ce00)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 4339 [select]:
net/http.(*persistConn).writeLoop(0xc00d1e46c0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2851129 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388495f48, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc023fe8a00, 0xc003030000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc023fe8a00, {0xc003030000, 0x37b9, 0x37b9})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc023fe8a00, {0xc003030000, 0x37b4, 0xc009125bc0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc001854530, {0xc003030000, 0xc003030000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc018833590, {0xc003030000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0567a13f8, {0x3544be0, 0xc018833590})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0567a1180, {0x7fc0b9166040, 0xc02af98480}, 0x37b9)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0567a1180, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0567a1180, {0xc01b911000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc0570ad320, {0xc0255bbd20, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc0570ad320}, {0xc0255bbd20, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0255bbd20, 0x9, 0xc0018252c0}, {0x35449e0, 0xc0570ad320})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0255bbce0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00e954f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc02dcb0480)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2839709 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc02da82048, 0x27)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x1851)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc02da82030, {0xc01fe40001, 0x1dff, 0x1dff})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x1852}, {0xc01fe40001, 0xc01fe40000, 0xc055d47cf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc038b1e280)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc038b1e280)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc038b1e280, {0x2748340, 0xc016bf8a20})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc03a5db6e0, {0xc04c0f6c00, 0x2000, 0x2600})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc038b48370, 0xc027b2ec80, {0x357ac00, 0xc03a66a580})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc03a5cdc00)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc03a5e0780)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 4000 [select]:
net/http.(*persistConn).writeLoop(0xc006800240)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1745841 [select]:
net/http.(*persistConn).writeLoop(0xc01fdc4d80)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2506765 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06549d110, 0xc0338a3200)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 701052 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc059d07110, 0xc058900c60)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 4235 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9125b50, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc014e71780, 0xc014e01000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc014e71780, {0xc014e01000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc014e71780, {0xc014e01000, 0xc06d2d8500, 0xc064024f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05d6cc038, {0xc014e01000, 0xc05614c680, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc014fbb440, {0xc014e01000, 0xc04bae1b60, 0xc064024d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc020b51080)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc020b51080, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc014fbb440)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2373590 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916b868, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc053109c00, 0xc05c7b3600, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc053109c00, {0xc05c7b3600, 0x81e, 0x81e})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc053109c00, {0xc05c7b3600, 0x7de, 0xc01f33e980})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc01a45ede8, {0xc05c7b3600, 0xc05c7b363b, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc022292fd8, {0xc05c7b3600, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc007678978, {0x3544be0, 0xc022292fd8})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc007678700, {0x7fc0b9166040, 0xc01f5f8100}, 0x7e3)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc007678700, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc007678700, {0xc01f372000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc05842eae0, {0xc05d7650e0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc05842eae0}, {0xc05d7650e0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc05d7650e0, 0x9, 0xc001066390}, {0x35449e0, 0xc05842eae0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc05d7650a0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc002bdef98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01bee8480)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 4279 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b91255e0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00d1bc100, 0xc03058c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00d1bc100, {0xc03058c000, 0x3c79, 0x3c79})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00d1bc100, {0xc03058c000, 0x2c55, 0xc00d1213e0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05d6cc9c0, {0xc03058c000, 0xc03058d01f, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0166d9590, {0xc03058c000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc055f31078, {0x3544be0, 0xc0166d9590})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc055f30e00, {0x7fc0b9166040, 0xc015c8fb40}, 0x2c5a)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc055f30e00, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc055f30e00, {0xc00d1fd000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc02bcfe480, {0xc01a1f0040, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc02bcfe480}, {0xc01a1f0040, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc01a1f0040, 0x9, 0xc00143ca50}, {0x35449e0, 0xc02bcfe480})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc01a1f0000)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc05ac79f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00d174d80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 4434 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917f710, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02102d800, 0xc05c7b1200, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02102d800, {0xc05c7b1200, 0x81e, 0x81e})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02102d800, {0xc05c7b1200, 0x7de, 0xc02112bc60})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc063f6e4d0, {0xc05c7b1200, 0xc05c7b123b, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc017441c50, {0xc05c7b1200, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc002bd9778, {0x3544be0, 0xc017441c50})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc002bd9500, {0x7fc0b9166040, 0xc01eed2d80}, 0x7e3)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc002bd9500, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc002bd9500, {0xc021145000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc021137500, {0xc01a3c11c0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc021137500}, {0xc01a3c11c0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc01a3c11c0, 0x9, 0xc001bcd5f0}, {0x35449e0, 0xc021137500})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc01a3c1180)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc059004f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01bdb6d80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 4372 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9125898, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc01bb3e080, 0xc01bd8a000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc01bb3e080, {0xc01bd8a000, 0x3c7b, 0x3c7b})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc01bb3e080, {0xc01bd8a000, 0x1c38, 0xc018b8f640})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05d6cca78, {0xc01bd8a000, 0xc01bd8c03e, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc01725dbd8, {0xc01bd8a000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0579e5778, {0x3544be0, 0xc01725dbd8})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0579e5500, {0x7fc0b9166040, 0xc01bb285c0}, 0x1c3d)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0579e5500, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0579e5500, {0xc01bd1c000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc03c04c180, {0xc01a1f0200, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc03c04c180}, {0xc01a1f0200, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc01a1f0200, 0x9, 0xc001bcd140}, {0x35449e0, 0xc03c04c180})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc01a1f01c0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc055fbbf98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00d175080)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 4394 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917f458, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02528f500, 0xc0218a9000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02528f500, {0xc0218a9000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02528f500, {0xc0218a9000, 0xc06d2d8b40, 0xc00c308f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc063f6e4f8, {0xc0218a9000, 0xc055fe9380, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0244425a0, {0xc0218a9000, 0xc0251fa3c0, 0xc00c308d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc023cc9ce0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc023cc9ce0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0244425a0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 4437 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917f628, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc01c7a5f80, 0xc0319b4000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc01c7a5f80, {0xc0319b4000, 0x3c7b, 0x3c7b})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc01c7a5f80, {0xc0319b4000, 0x3c76, 0xc02112be80})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05d6ccae8, {0xc0319b4000, 0xc0319b4000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc017147bc0, {0xc0319b4000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc006976978, {0x3544be0, 0xc017147bc0})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc006976700, {0x7fc0b9166040, 0xc021b16480}, 0x3c7b)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc006976700, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc006976700, {0xc02115d000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc021158ba0, {0xc01a3c1380, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc021158ba0}, {0xc01a3c1380, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc01a3c1380, 0x9, 0xc0012416e0}, {0x35449e0, 0xc021158ba0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc01a3c1340)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00d808f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01bdb7080)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2845370 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc0390f7b48, 0x3)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x86)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc0390f7b30, {0xc00cdcd201, 0x5ff, 0x5ff})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x87}, {0xc00cdcd201, 0xc00cdcd200, 0xc055d41cf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc039520c80)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc039520c80)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc039520c80, {0x2748340, 0xc01b5a1bd8})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0395194d0, {0xc03954d400, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc039505ef0, 0xc001c63ea0, {0x357ac00, 0xc014ad2140})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0394e3720)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc03952e400)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2640461 [select]:
net/http.(*persistConn).writeLoop(0xc001d6e120)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 693287 [select]:
net/http.(*persistConn).writeLoop(0xc029ff18c0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2856566 [select]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc03abfd200, 0xc00220e900)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc033916ea0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 4414 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9180e18, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc033626900, 0xc0325a8000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc033626900, {0xc0325a8000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc033626900, {0xc0325a8000, 0xc06d2d8f00, 0xc00755bf68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc05d6ccb38, {0xc0325a8000, 0xc058ccdd40, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc033493320, {0xc0325a8000, 0xc033233200, 0xc00755bd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc0324733e0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc0324733e0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc033493320)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 4415 [select]:
net/http.(*persistConn).writeLoop(0xc033493320)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2553153 [select]:
net/http.(*persistConn).writeLoop(0xc01c2d0fc0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2504729 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185d6410, 0xc021508f00)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 695907 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916b698, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc036770d80, 0xc036b51000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc036770d80, {0xc036b51000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc036770d80, {0xc036b51000, 0xc06d2d8b40, 0xc000e80f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc03a47dea0, {0xc036b51000, 0xc030cd5860, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc02b9170e0, {0xc036b51000, 0xc031b607e0, 0xc000e80d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc036b96840)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc036b96840, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc02b9170e0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2842683 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc013dc61c8, 0x3)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x86)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc013dc61b0, {0xc000474601, 0x5ff, 0x5ff})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x87}, {0xc000474601, 0xc000474600, 0xc055f17cf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc0288c7a40)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc0288c7a40)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc0288c7a40, {0x2748340, 0xc01fdb06a8})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0245ea3c0, {0xc01ded6c00, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc013cdbf90, 0xc052df43f0, {0x357ac00, 0xc020701840})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc013c9df40)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc036838840)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 4537 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9180990, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc03532eb00, 0xc035f9f000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc03532eb00, {0xc035f9f000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc03532eb00, {0xc035f9f000, 0xc06d2d8640, 0xc00d6f4f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00ba38e28, {0xc035f9f000, 0xc06dab96c0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc035362240, {0xc035f9f000, 0xc035925440, 0xc00d6f4d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc03595fb60)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03595fb60, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc035362240)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2853789 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc01b6d84c8, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc01b6d84b8)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc01b6d84b0, {0xc01b2b3c00, 0x200, 0x200})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x4db95e0}, {0xc01b2b3c00, 0x217c472, 0xc055320cf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc022b4d540)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc022b4d540)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc022b4d540, {0x2748340, 0xc01b841938})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc01b64be00, {0xc01b643c00, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc01b644b90, 0xc055320e38, {0x357ac00, 0xc01b834d00})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc01b614ec0)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc01b834cc0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2851784 [select, 2 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc02266de00, 0xc03c2f5900)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc024180000)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 2553038 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020db6b60, 0xc018b7bf40)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2507281 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9187b38, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc057afac80, 0xc00c836000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc057afac80, {0xc00c836000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc057afac80, {0xc00c836000, 0xc06d2d8640, 0xc003084f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc032c1ded8, {0xc00c836000, 0xc0274ce680, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc01deadb00, {0xc00c836000, 0xc04ae31a40, 0xc003084d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc01dff3f80)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc01dff3f80, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc01deadb00)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 693960 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc016fa4ea0, 0xc045c635c0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2785909 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7fc0b9187d08, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc020aab700, 0xc02b915000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc020aab700, {0xc02b915000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc020aab700, {0xc02b915000, 0x4d9ca6, 0x4d8e460})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00ee68bf0, {0xc02b915000, 0x0, 0x0})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).Read(0xc005f69830, {0xc02b915000, 0x1000, 0x1000})
	/usr/local/go/src/net/http/server.go:780 +0x16d
bufio.(*Reader).fill(0xc02bbc6120)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc02bbc6120, 0x4)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*conn).serve(0xc00d954d20, {0x35a0d38, 0xc000f2f740})
	/usr/local/go/src/net/http/server.go:1955 +0xc36
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3034 +0x4e8

goroutine 1215098 [select]:
net/http.(*persistConn).writeLoop(0xc011052240)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 37736 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917d858, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc066c9ba80, 0xc066281000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc066c9ba80, {0xc066281000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc066c9ba80, {0xc066281000, 0xc06d2d9040, 0xc05900ff68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc019d57538, {0xc066281000, 0xc04838ab60, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc066b9fc20, {0xc066281000, 0xc067459e60, 0xc05900fd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc065de8ba0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc065de8ba0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc066b9fc20)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2504569 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc031d5ca90, 0xc04bed2030)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2554358 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916efb8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05e12b700, 0xc018e46000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05e12b700, {0xc018e46000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05e12b700, {0xc018e46000, 0xc06d2d8b40, 0xc00c306f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00af9c1a8, {0xc018e46000, 0xc01a549a00, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0106db560, {0xc018e46000, 0xc024d32960, 0xc00c306d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc03db31020)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc03db31020, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0106db560)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2849363 [sync.Cond.Wait, 1 minutes]:
sync.runtime_notifyListWait(0xc024ef3cc8, 0x3)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x86)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc024ef3cb0, {0xc036c88001, 0x1dff, 0x1dff})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x87}, {0xc036c88001, 0xc036c88000, 0xc05516bcf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc033dd1180)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc033dd1180)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc033dd1180, {0x2748340, 0xc01ef11530})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0360bb7d0, {0xc04c0fb800, 0x2000, 0x2600})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0360f2e10, 0xc01a21f260, {0x357ac00, 0xc0219a31c0})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc035b7bcc0)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc035bbee40)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2842737 [select, 5 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc013dc6480, 0xc0149c6b00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc0245eb290)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 2841063 [select, 5 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc01b79de00, 0xc00ff99300)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a0d38, 0xc000f2f740)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 2367688 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc02e936000, 0xc00f84e6c0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 704022 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b91876b0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc057771100, 0xc056c49000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc057771100, {0xc056c49000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc057771100, {0xc056c49000, 0xc06d2d8f00, 0xc00ed17f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0005eb450, {0xc056c49000, 0xc04838aea0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc056cafb00, {0xc056c49000, 0xc045c636e0, 0xc00ed17d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc056f86cc0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc056f86cc0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc056cafb00)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2733846 [select]:
net/http.(*persistConn).writeLoop(0xc05cccb9e0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1215097 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388495468, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05edbd080, 0xc01fd1d000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05edbd080, {0xc01fd1d000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05edbd080, {0xc01fd1d000, 0xc06d2d8140, 0xc005bbff68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00af9c1b0, {0xc01fd1d000, 0xc02b4d3520, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc011052240, {0xc01fd1d000, 0xc04e1b4660, 0xc005bbfd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc006163020)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc006163020, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc011052240)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2713013 [select]:
net/http.(*persistConn).writeLoop(0xc0016dc900)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2851660 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917fb98, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc058693900, 0xc010ed2000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc058693900, {0xc010ed2000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc058693900, {0xc010ed2000, 0xc06d2d8f00, 0xc05900ef68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc009d86980, {0xc010ed2000, 0xc055fe89c0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0214d4900, {0xc010ed2000, 0xc077bd5620, 0xc05900ed30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc02b899d40)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc02b899d40, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0214d4900)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2857697 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0161de648, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc0161de638)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc0161de630, {0xc0161cd800, 0x200, 0x200})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc0161cd800, 0xc055d18560, 0xc055d184f0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc035235680)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc035235680)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc035235680, {0x2748340, 0xc017367ba8})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc02204ca80, {0xc018f7cc00, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc016c16cd0, 0x50, {0x357ac00, 0xc02dc342c0})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc017299720)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc02dc34280)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2852422 [select]:
net/http.(*persistConn).writeLoop(0xc05f585e60)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2848493 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc017595040, 0xc01cd646c0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2845369 [select, 4 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc0390f7b00, 0xc0394c5800)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a0d38, 0xc000f2f740)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 695986 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b91257b0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0417be400, 0xc041475000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0417be400, {0xc041475000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0417be400, {0xc041475000, 0xc06d2d8f00, 0xc007446f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc03b7fa560, {0xc041475000, 0xc06851a680, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0532ee480, {0xc041475000, 0xc07717c5a0, 0xc007446d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc041ed7f20)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc041ed7f20, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0532ee480)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2831478 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc01d001b48, 0x5)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x86)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc01d001b30, {0xc0268f0001, 0x1dff, 0x1dff})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x87}, {0xc0268f0001, 0xc0268f0000, 0xc055ebfcf0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc022a0ac80)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc022a0ac80)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc022a0ac80, {0x2748340, 0xc017992f48})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc01fbae300, {0xc026e38000, 0x2000, 0x2600})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc01f853770, 0xc000947880, {0x357ac00, 0xc023020d80})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc01f9226c0)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc01fb01340)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x107
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 2553399 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b91810d0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc026005e80, 0xc02b826000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc026005e80, {0xc02b826000, 0x3cd9, 0x3cd9})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc026005e80, {0xc02b826000, 0x2cb5, 0xc02665c6e0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00a095e78, {0xc02b826000, 0xc02b82701f, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc024fa43a8, {0xc02b826000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc01dea53f8, {0x3544be0, 0xc024fa43a8})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc01dea5180, {0x7fc0b9166040, 0xc02654d900}, 0x2cba)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc01dea5180, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc01dea5180, {0xc029104000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc03d8812c0, {0xc01ebf7b60, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc03d8812c0}, {0xc01ebf7b60, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc01ebf7b60, 0x9, 0xc001f77a40}, {0x35449e0, 0xc03d8812c0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc01ebf7b20)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00d6f7f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc026609500)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 705999 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b91874e0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc04d25ef00, 0xc05c473b00, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc04d25ef00, {0xc05c473b00, 0x81c, 0x81c})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc04d25ef00, {0xc05c473b00, 0x7dc, 0xc02a9e5220})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc03b410190, {0xc05c473b00, 0xc05c473b3b, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0166cc5d0, {0xc05c473b00, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0049a0cf8, {0x3544be0, 0xc0166cc5d0})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0049a0a80, {0x7fc0b9166040, 0xc0554d5540}, 0x7e1)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0049a0a80, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0049a0a80, {0xc02acab000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc02ac9c780, {0xc02a7fd1c0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc02ac9c780}, {0xc02a7fd1c0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc02a7fd1c0, 0x9, 0xc00143c210}, {0x35449e0, 0xc02ac9c780})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc02a7fd180)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc036caaf98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc02ac77b00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 693991 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc000ec9860, 0xc05d9bb700)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2847503 [select, 3 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc03a69e180, 0xc03a42cd00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc03a6922d0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 2508601 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917fc80, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0148ce080, 0xc0479d3000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0148ce080, {0xc0479d3000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0148ce080, {0xc0479d3000, 0xc06d2d8f00, 0xc003087f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc002066bd0, {0xc0479d3000, 0xc019da04e0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc01f7687e0, {0xc0479d3000, 0xc039fd6060, 0xc003087d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc02e5659e0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc02e5659e0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc01f7687e0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 705224 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b91875c8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc04d4f7680, 0xc02eca8000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc04d4f7680, {0xc02eca8000, 0x3c79, 0x3c79})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc04d4f7680, {0xc02eca8000, 0x2c55, 0xc06aced420})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc029b42a88, {0xc02eca8000, 0xc02eca901f, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc035820468, {0xc02eca8000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc05b58f078, {0x3544be0, 0xc035820468})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc05b58ee00, {0x7fc0b9166040, 0xc0827e0980}, 0x2c5a)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc05b58ee00, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc05b58ee00, {0xc036a7b000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc021fdbe60, {0xc01b495380, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc021fdbe60}, {0xc01b495380, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc01b495380, 0x9, 0xc00100e4b0}, {0x35449e0, 0xc021fdbe60})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc01b495340)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc051012f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01ad91980)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 695084 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916b040, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc03f43bc00, 0xc019650000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc03f43bc00, {0xc019650000, 0x3c7b, 0x3c7b})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc03f43bc00, {0xc019650000, 0x3c76, 0xc02f97ec20})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc014f54208, {0xc019650000, 0xc019650000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc013b57650, {0xc019650000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0214cfe78, {0x3544be0, 0xc013b57650})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0214cfc00, {0x7fc0b9166040, 0xc028699580}, 0x3c7b)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0214cfc00, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0214cfc00, {0xc020fa7000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc02b492900, {0xc012796f20, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc02b492900}, {0xc012796f20, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc012796f20, 0x9, 0xc001c3a510}, {0x35449e0, 0xc02b492900})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc012796ee0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00d80bf98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc01279cc00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 695908 [select]:
net/http.(*persistConn).writeLoop(0xc02b9170e0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1786914 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc014886680, 0xc013e40980)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2367747 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0302600d0, 0xc014506d80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2715715 [select]:
net/http.(*persistConn).writeLoop(0xc02136d560)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2715020 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06549cb60, 0xc00050a7a0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 704023 [select]:
net/http.(*persistConn).writeLoop(0xc056cafb00)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1744004 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916e878, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc04d4f6e00, 0xc02bdd5000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc04d4f6e00, {0xc02bdd5000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc04d4f6e00, {0xc02bdd5000, 0xc06d2d9540, 0xc059011f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00a230098, {0xc02bdd5000, 0xc05614da00, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc04f452c60, {0xc02bdd5000, 0xc01a802840, 0xc059011d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc0580d7d40)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc0580d7d40, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc04f452c60)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1744951 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9054b58, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc02c127c80, 0xc026a3f000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc02c127c80, {0xc026a3f000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc02c127c80, {0xc026a3f000, 0xc06d2d8dc0, 0xc06402bf68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0131d0468, {0xc026a3f000, 0xc03d883380, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc005f5bd40, {0xc026a3f000, 0xc00157f440, 0xc06402bd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc045ab9c80)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc045ab9c80, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc005f5bd40)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2732952 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388495c90, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0249cb600, 0xc02790a000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0249cb600, {0xc02790a000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0249cb600, {0xc02790a000, 0xc06d2d8b40, 0xc000db3f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0130be6a0, {0xc02790a000, 0xc04f6e3d40, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc02a88fd40, {0xc02790a000, 0xc0008ff0e0, 0xc000db3d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc053fdd5c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc053fdd5c0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc02a88fd40)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1205516 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc02afceb60, 0xc040343a80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 703857 [select]:
net/http.(*persistConn).writeLoop(0xc05665e240)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2684220 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917d4b8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc036083680, 0xc02d43d000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc036083680, {0xc02d43d000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc036083680, {0xc02d43d000, 0xc06d2d8f00, 0xc059001f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc028f6e148, {0xc02d43d000, 0xc029b944e0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc032adcc60, {0xc02d43d000, 0xc00baec5a0, 0xc059001d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc02b1e1500)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc02b1e1500, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc032adcc60)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2684343 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9180d30, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc056968800, 0xc02805d000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc056968800, {0xc02805d000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc056968800, {0xc02805d000, 0xc06d2d8b40, 0xc00338af68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc024cfe560, {0xc02805d000, 0xc03d883a00, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc05caade60, {0xc02805d000, 0xc00159b6e0, 0xc00338ad30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc040344720)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc040344720, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc05caade60)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 37737 [select]:
net/http.(*persistConn).writeLoop(0xc066b9fc20)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1214812 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc03176e5b0, 0xc03cb50c00)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 700960 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc05f41f5f0, 0xc04cbfd500)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2367662 [select]:
net/http.(*persistConn).writeLoop(0xc000f79200)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1744952 [select]:
net/http.(*persistConn).writeLoop(0xc005f5bd40)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2716404 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916ec18, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0589ef300, 0xc001dcc000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0589ef300, {0xc001dcc000, 0x3cd7, 0x3cd7})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0589ef300, {0xc001dcc000, 0x3cd2, 0xc01faab560})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc023184b78, {0xc001dcc000, 0xc001dcc000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc024fa5ef0, {0xc001dcc000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc01f3a05f8, {0x3544be0, 0xc024fa5ef0})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc01f3a0380, {0x7fc0b9166040, 0xc01cd64600}, 0x3cd7)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc01f3a0380, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc01f3a0380, {0xc01da91000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc0261d3d40, {0xc04399fee0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc0261d3d40}, {0xc04399fee0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc04399fee0, 0x9, 0xc001fe2240}, {0x35449e0, 0xc0261d3d40})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc04399fea0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc002bdcf98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc02968c300)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 695971 [select]:
net/http.(*persistConn).writeLoop(0xc02ba84b40)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2639543 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01bb72a90, 0xc008ea2600)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 703704 [IO wait]:
internal/poll.runtime_pollWait(0x7fc388496030, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc037034a00, 0xc0432fd000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc037034a00, {0xc0432fd000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc037034a00, {0xc0432fd000, 0xc06d2d8640, 0xc051017f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc02fa74080, {0xc0432fd000, 0xc02b4d36c0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc050f23440, {0xc0432fd000, 0xc065ad5500, 0xc051017d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc050e90900)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc050e90900, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc050f23440)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2684126 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917d5a0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc024d92700, 0xc02d433000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc024d92700, {0xc02d433000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc024d92700, {0xc02d433000, 0xc06d2d8c80, 0xc003082f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc028f6e130, {0xc02d433000, 0xc029b95380, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc026593e60, {0xc02d433000, 0xc00baec300, 0xc003082d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc02b1e12c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc02b1e12c0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc026593e60)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2370199 [select]:
net/http.(*persistConn).writeLoop(0xc017668360)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1744005 [select]:
net/http.(*persistConn).writeLoop(0xc04f452c60)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2851033 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0587bd860, 0xc02f7b3440)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 703856 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9187880, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc06202ed00, 0xc0510cd000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc06202ed00, {0xc0510cd000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc06202ed00, {0xc0510cd000, 0xc06d2d8b40, 0xc055ee2f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc023184a78, {0xc0510cd000, 0xc019da0820, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc05665e240, {0xc0510cd000, 0xc022ef6420, 0xc055ee2d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc0516f1aa0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc0516f1aa0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc05665e240)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2505474 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916a9e8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0445e4880, 0xc05c474d00, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0445e4880, {0xc05c474d00, 0x820, 0x820})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0445e4880, {0xc05c474d00, 0x81b, 0xc00c087060})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0346ccc30, {0xc05c474d00, 0xc05c474d00, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc018833770, {0xc05c474d00, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc005398cf8, {0x3544be0, 0xc018833770})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc005398a80, {0x7fc0b9166040, 0xc02e369fc0}, 0x820)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc005398a80, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc005398a80, {0xc00ce76000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc01e3c0240, {0xc035697a80, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc01e3c0240}, {0xc035697a80, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc035697a80, 0x9, 0xc0018257d0}, {0x35449e0, 0xc01e3c0240})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc035697a40)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc05ac77f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc012d54180)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 693958 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc016fa4a90, 0xc046783040)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2639908 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916eb30, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc056963e00, 0xc03090a000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc056963e00, {0xc03090a000, 0x3c7d, 0x3c7d})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc056963e00, {0xc03090a000, 0x3c78, 0xc022dbd940})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc01143d8b0, {0xc03090a000, 0xc03090a000, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc022292f60, {0xc03090a000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc00a5e3778, {0x3544be0, 0xc022292f60})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc00a5e3500, {0x7fc0b9166040, 0xc02171e980}, 0x3c7d)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc00a5e3500, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc00a5e3500, {0xc0376ab000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc040329d40, {0xc038622d60, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc040329d40}, {0xc038622d60, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc038622d60, 0x9, 0xc0010661e0}, {0x35449e0, 0xc040329d40})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc038622d20)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc005bc4f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00a620780)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2684021 [select]:
net/http.(*persistConn).writeLoop(0xc0190527e0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2506201 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc02299c410, 0xc03d52d320)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2638744 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b9125328, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc036771e80, 0xc053670d00, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc036771e80, {0xc053670d00, 0x820, 0x820})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc036771e80, {0xc053670d00, 0x7e0, 0xc020b15620})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0140b4680, {0xc053670d00, 0xc053670d3b, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc017367728, {0xc053670d00, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc00cfebaf8, {0x3544be0, 0xc017367728})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc00cfeb880, {0x7fc0b9166040, 0xc014d68780}, 0x7e5)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc00cfeb880, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc00cfeb880, {0xc03d9c7000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc0603f4960, {0xc0385a8ba0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc0603f4960}, {0xc0385a8ba0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0385a8ba0, 0x9, 0xc00104c360}, {0x35449e0, 0xc0603f4960})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0385a8b60)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc055ecff98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00d1e8900)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2684020 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917f1a0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0583f3380, 0xc018a35000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0583f3380, {0xc018a35000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0583f3380, {0xc018a35000, 0xc06d2d8640, 0xc00338ff68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc024cff098, {0xc018a35000, 0xc000eb6ea0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0190527e0, {0xc018a35000, 0xc010857860, 0xc00338fd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc018aded80)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc018aded80, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0190527e0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2839735 [select, 5 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc02da82000, 0xc03a1c1e00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc02a797b00)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 694514 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e250270, 0xc038045980)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2716229 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b917d3d0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0258d1c00, 0xc02c61c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0258d1c00, {0xc02c61c000, 0x3cd7, 0x3cd7})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0258d1c00, {0xc02c61c000, 0x3cd2, 0x413b05})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0352053c0, {0xc02c61c000, 0x7fc0afe537f8, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0250131e8, {0xc02c61c000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc01b677778, {0x3544be0, 0xc0250131e8})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc01b677500, {0x7fc0b9166040, 0xc0260a2d40}, 0x3cd7)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc01b677500, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc01b677500, {0xc032118000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc032114ba0, {0xc0378bcd60, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc032114ba0}, {0xc0378bcd60, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0378bcd60, 0x9, 0xc0010185a0}, {0x35449e0, 0xc032114ba0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0378bcd20)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc000db1f98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc0260fc180)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 693286 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916f358, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc055fd2600, 0xc02c244000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc055fd2600, {0xc02c244000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc055fd2600, {0xc02c244000, 0xc06d2d8c80, 0xc00e950f68})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc01152f110, {0xc02c244000, 0xc055f2fba0, 0x4})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc029ff18c0, {0xc02c244000, 0xc01d8c2a20, 0xc00e950d30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc07774a3c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc07774a3c0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc029ff18c0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 2639562 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020c695f0, 0xc058f89980)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 37013 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0393a2000, 0xc02fcf2a40)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 1744088 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc00c325110, 0x0)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 701051 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc059d07040, 0xc03f5bfb80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 747624 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc03f721040, 0xc063caea80)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2551279 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0148860d0, 0xc0195e3b00)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2734983 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc03dfd3380, 0xc0120e6900)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2687356 [IO wait]:
internal/poll.runtime_pollWait(0x7fc0b916b128, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05e304180, 0xc006ac8000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05e304180, {0xc006ac8000, 0x3cd7, 0x3cd7})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05e304180, {0xc006ac8000, 0x2cb3, 0xc0004edd20})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc02df12658, {0xc006ac8000, 0xc006ac901f, 0x5})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc05fcac150, {0xc006ac8000, 0x0, 0x409b6d})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc015cf6978, {0x3544be0, 0xc05fcac150})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc015cf6700, {0x7fc0b9166040, 0xc020c07ec0}, 0x2cb8)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc015cf6700, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc015cf6700, {0xc021ca2000, 0x1000, 0x8d4340})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc04ebcccc0, {0xc0205e98c0, 0x9, 0x8e2302})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x35449e0, 0xc04ebcccc0}, {0xc0205e98c0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0205e98c0, 0x9, 0xc00161ebd0}, {0x35449e0, 0xc04ebcccc0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0205e9880)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc055d6ef98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc02bd68300)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 2853421 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7fc388495380, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc05ed86280, 0xc010eca000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc05ed86280, {0xc010eca000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc05ed86280, {0xc010eca000, 0x40, 0xc03885c600})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc009e67e28, {0xc010eca000, 0xc053987400, 0xc01afe7000})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).Read(0xc01afe9740, {0xc010eca000, 0x1000, 0x1000})
	/usr/local/go/src/net/http/server.go:780 +0x16d
bufio.(*Reader).fill(0xc02b8999e0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).ReadSlice(0xc02b8999e0, 0x0)
	/usr/local/go/src/bufio/bufio.go:360 +0x2f
bufio.(*Reader).ReadLine(0xc02b8999e0)
	/usr/local/go/src/bufio/bufio.go:389 +0x27
net/textproto.(*Reader).readLineSlice(0xc022b1fb00)
	/usr/local/go/src/net/textproto/reader.go:57 +0x99
net/textproto.(*Reader).ReadLine(...)
	/usr/local/go/src/net/textproto/reader.go:38
net/http.readRequest(0xc01afe8cf0)
	/usr/local/go/src/net/http/request.go:1029 +0x79
net/http.(*conn).readRequest(0xc01afe4dc0, {0x35a0c90, 0xc01afe6fc0})
	/usr/local/go/src/net/http/server.go:966 +0x225
net/http.(*conn).serve(0xc01afe4dc0, {0x35a0d38, 0xc000f2f740})
	/usr/local/go/src/net/http/server.go:1856 +0x865
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3034 +0x4e8

goroutine 1782337 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01bb72ea0, 0xc053108300)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2637835 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc036d22340, 0xc03ff34960)
	/app/scrape/scrape.go:1205 +0x425
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:584 +0xa55

goroutine 2508602 [select]:
net/http.(*persistConn).writeLoop(0xc01f7687e0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2855062 [select, 1 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc0091e8f00, 0xc00e4ec800)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaae
golang.org/x/net/http2.(*clientStream).doRequest(0x35a16a0, 0xc0283c8980)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 2733691 [select]:
net/http.(*persistConn).writeLoop(0xc06a763200)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65