prometheus_out_of_mem.log

fatal error: runtime: out of memory

runtime stack:
runtime.throw(0x253885d, 0x16)
#011/usr/local/go/src/runtime/panic.go:774 +0x72
runtime.sysMap(0xc1c0000000, 0x4000000, 0x3f5bc78)
#011/usr/local/go/src/runtime/mem_linux.go:169 +0xc5
runtime.(*mheap).sysAlloc(0x3f432c0, 0x172000, 0x128000, 0x0)
#011/usr/local/go/src/runtime/malloc.go:701 +0x1cd
runtime.(*mheap).grow(0x3f432c0, 0xb9, 0xffffffff)
#011/usr/local/go/src/runtime/mheap.go:1255 +0xa3
runtime.(*mheap).allocSpanLocked(0x3f432c0, 0xb9, 0x3f5bc88, 0x20306f00000000)
#011/usr/local/go/src/runtime/mheap.go:1170 +0x266
runtime.(*mheap).alloc_m(0x3f432c0, 0xb9, 0x101, 0x7f202d3f1fff)
#011/usr/local/go/src/runtime/mheap.go:1022 +0xc2
runtime.(*mheap).alloc.func1()
#011/usr/local/go/src/runtime/mheap.go:1093 +0x4c
runtime.(*mheap).alloc(0x3f432c0, 0xb9, 0x101, 0x200)
#011/usr/local/go/src/runtime/mheap.go:1092 +0x8a
runtime.largeAlloc(0x172000, 0xc000060100, 0x45b3ea)
#011/usr/local/go/src/runtime/malloc.go:1138 +0x97
runtime.mallocgc.func1()
#011/usr/local/go/src/runtime/malloc.go:1033 +0x46
runtime.systemstack(0x0)
#011/usr/local/go/src/runtime/asm_amd64.s:370 +0x66
runtime.mstart()
#011/usr/local/go/src/runtime/proc.go:1146

goroutine 2835 [running]:
runtime.systemstack_switch()
#011/usr/local/go/src/runtime/asm_amd64.s:330 fp=0xc000abdc70 sp=0xc000abdc68 pc=0x45d180
runtime.mallocgc(0x172000, 0x0, 0x45f000, 0x3eb4c40)
#011/usr/local/go/src/runtime/malloc.go:1032 +0x895 fp=0xc000abdd10 sp=0xc000abdc70 pc=0x40c755
runtime.growslice(0x226e860, 0xc1bfec8000, 0xc555, 0xc555, 0xc556, 0xc1bfec8000, 0x9d55, 0xc555)
#011/usr/local/go/src/runtime/slice.go:175 +0x14e fp=0xc000abdd78 sp=0xc000abdd10 pc=0x445d8e
github.com/prometheus/prometheus/tsdb/record.(*Decoder).Samples(0x3f59ca8, 0xc1b9796000, 0xb0788, 0x2a2000, 0x3f59ca8, 0xc555, 0xc555, 0xc19ad97740, 0x5, 0x8, ...)
#011/app/tsdb/record/record.go:128 +0x23b fp=0xc000abde60 sp=0xc000abdd78 pc=0x1bfafbb
github.com/prometheus/prometheus/tsdb.(*Head).loadWAL.func6(0xc197fe7d40, 0xc1b8d70000, 0x3f59ca8, 0xc185997aa0, 0xc00021ae40, 0xc197fe7da0, 0xc185997ad0, 0xc185997b00)
#011/app/tsdb/head.go:438 +0x628 fp=0xc000abdfa0 sp=0xc000abde60 pc=0x1c47468
runtime.goexit()
#011/usr/local/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc000abdfa8 sp=0xc000abdfa0 pc=0x45f0d1
created by github.com/prometheus/prometheus/tsdb.(*Head).loadWAL
#011/app/tsdb/head.go:419 +0x4b6

goroutine 1 [chan receive, 7 minutes]:
github.com/oklog/run.(*Group).Run(0xc000ae7c00, 0xc00010a810, 0x8)
#011/app/vendor/github.com/oklog/run/group.go:43 +0xed
main.main()
#011/app/cmd/prometheus/main.go:739 +0x6c91

goroutine 17 [syscall, 7 minutes]:
os/signal.signal_recv(0x0)
#011/usr/local/go/src/runtime/sigqueue.go:147 +0x9c
os/signal.loop()
#011/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
#011/usr/local/go/src/os/signal/signal_unix.go:29 +0x41

goroutine 13 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0000bcaf0)
#011/app/vendor/go.opencensus.io/stats/view/worker.go:154 +0x100
created by go.opencensus.io/stats/view.init.0
#011/app/vendor/go.opencensus.io/stats/view/worker.go:32 +0x57

goroutine 54 [chan receive, 7 minutes]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc0000d7600)
#011/app/vendor/google.golang.org/grpc/clientconn.go:1078 +0x6b0
created by google.golang.org/grpc.(*addrConn).connect
#011/app/vendor/google.golang.org/grpc/clientconn.go:744 +0x120

goroutine 105 [select]:
github.com/prometheus/prometheus/pkg/logging.(*Deduper).run(0xc00009b340)
#011/app/pkg/logging/dedupe.go:75 +0x1f3
created by github.com/prometheus/prometheus/pkg/logging.Dedupe
#011/app/pkg/logging/dedupe.go:61 +0xcf

goroutine 106 [chan receive]:
github.com/prometheus/prometheus/storage/remote.(*WriteStorage).run(0xc000084cc0)
#011/app/storage/remote/write.go:79 +0xc7
created by github.com/prometheus/prometheus/storage/remote.NewWriteStorage
#011/app/storage/remote/write.go:72 +0x122

goroutine 116 [select, 7 minutes]:
main.main.func6(0x0, 0x0)
#011/app/cmd/prometheus/main.go:505 +0xfa
github.com/oklog/run.(*Group).Run.func1(0xc0002409c0, 0xc000117600, 0xc00010a670)
#011/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
#011/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 117 [chan receive, 7 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).Run(0xc00018e280, 0x0, 0x0)
#011/app/discovery/manager.go:171 +0x72
main.main.func8(0x0, 0x0)
#011/app/cmd/prometheus/main.go:525 +0x4e
github.com/oklog/run.(*Group).Run.func1(0xc0002409c0, 0xc00065e4a0, 0xc00065e4c0)
#011/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
#011/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 118 [chan receive, 7 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).Run(0xc00018e500, 0x0, 0x0)
#011/app/discovery/manager.go:171 +0x72
main.main.func10(0x0, 0x0)
#011/app/cmd/prometheus/main.go:539 +0x4e
github.com/oklog/run.(*Group).Run.func1(0xc0002409c0, 0xc00065e500, 0xc00065e520)
#011/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
#011/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 119 [chan receive, 7 minutes]:
main.main.func12(0x0, 0x0)
#011/app/cmd/prometheus/main.go:557 +0x6c
github.com/oklog/run.(*Group).Run.func1(0xc0002409c0, 0xc00063f3b0, 0xc00065e540)
#011/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
#011/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 120 [chan receive, 7 minutes]:
main.main.func14(0x0, 0x0)
#011/app/cmd/prometheus/main.go:581 +0x98
github.com/oklog/run.(*Group).Run.func1(0xc0002409c0, 0xc000240900, 0xc00010a6d0)
#011/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
#011/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 121 [select, 7 minutes]:
main.main.func16(0x0, 0x0)
#011/app/cmd/prometheus/main.go:614 +0xf0
github.com/oklog/run.(*Group).Run.func1(0xc0002409c0, 0xc000240960, 0xc00010a7e0)
#011/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
#011/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 122 [chan receive, 7 minutes]:
main.main.func18(0x0, 0x0)
#011/app/cmd/prometheus/main.go:645 +0x48
github.com/oklog/run.(*Group).Run.func1(0xc0002409c0, 0xc00065e580, 0xc00065e5a0)
#011/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
#011/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 123 [chan receive]:
github.com/prometheus/prometheus/tsdb.(*Head).loadWAL(0xc00012e000, 0xc1b8d70000, 0xc000309b40, 0xc1b8cf0000, 0x0)
#011/app/tsdb/head.go:471 +0x57f
github.com/prometheus/prometheus/tsdb.(*Head).Init(0xc00012e000, 0x16fb9cfc900, 0x0, 0x0)
#011/app/tsdb/head.go:618 +0x5bc
github.com/prometheus/prometheus/tsdb.Open(0x7ffef7279ee4, 0xb, 0x2c90040, 0xc00063f500, 0x2ccb760, 0xc00012a460, 0xc000117740, 0x2, 0xc000590380, 0xc)
#011/app/tsdb/db.go:575 +0x601
github.com/prometheus/prometheus/storage/tsdb.Open(0x7ffef7279ee4, 0xb, 0x2c90040, 0xc00063f500, 0x2ccb760, 0xc00012a460, 0xc0004b26e0, 0x0, 0x2c90600, 0x0)
#011/app/storage/tsdb/tsdb.go:192 +0x290
main.main.func20(0x2ccb5a0, 0x2c90de0)
#011/app/cmd/prometheus/main.go:667 +0x31a
github.com/oklog/run.(*Group).Run.func1(0xc0002409c0, 0xc000117680, 0xc00063f3e0)
#011/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
#011/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 124 [select, 7 minutes]:
github.com/prometheus/prometheus/web.(*Handler).Run(0xc0000f1600, 0x2ce1ee0, 0xc00009b400, 0x0, 0x0)
#011/app/web/web.go:585 +0x1048
main.main.func22(0x0, 0x0)
#011/app/cmd/prometheus/main.go:707 +0x40
github.com/oklog/run.(*Group).Run.func1(0xc0002409c0, 0xc00065e600, 0xc00010a800)
#011/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
#011/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 125 [chan receive, 7 minutes]:
main.main.func24(0x0, 0x0)
#011/app/cmd/prometheus/main.go:728 +0x6c
github.com/oklog/run.(*Group).Run.func1(0xc0002409c0, 0xc00063f410, 0xc00010a810)
#011/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
#011/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 126 [select]:
github.com/prometheus/prometheus/discovery.(*Manager).sender(0xc00018e280)
#011/app/discovery/manager.go:263 +0x123
created by github.com/prometheus/prometheus/discovery.(*Manager).Run
#011/app/discovery/manager.go:170 +0x43

goroutine 127 [select]:
github.com/prometheus/prometheus/discovery.(*Manager).sender(0xc00018e500)
#011/app/discovery/manager.go:263 +0x123
created by github.com/prometheus/prometheus/discovery.(*Manager).Run
#011/app/discovery/manager.go:170 +0x43

goroutine 156 [select, 7 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000219f00)
#011/app/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:115 +0x12c
created by google.golang.org/grpc.newCCBalancerWrapper
#011/app/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:106 +0x169

goroutine 157 [chan receive, 7 minutes]:
github.com/prometheus/prometheus/prompb.RegisterAdminHandlerFromEndpoint.func1.1(0x2ce1ee0, 0xc00009b400, 0xc000740380, 0x25222bb, 0xc)
#011/app/prompb/rpc.pb.gw.go:92 +0x4c
created by github.com/prometheus/prometheus/prompb.RegisterAdminHandlerFromEndpoint.func1
#011/app/prompb/rpc.pb.gw.go:91 +0x1b4

goroutine 158 [chan receive]:
github.com/soheilhy/cmux.muxListener.Accept(...)
#011/app/vendor/github.com/soheilhy/cmux/cmux.go:229
net/http.(*Server).Serve(0xc0004801c0, 0x2cd9360, 0xc0000fb140, 0x0, 0x0)
#011/usr/local/go/src/net/http/server.go:2896 +0x280
github.com/prometheus/prometheus/web.(*Handler).Run.func3(0xc000584e40, 0xc0004801c0, 0x2cd9360, 0xc0000fb140)
#011/app/web/web.go:576 +0x3f
created by github.com/prometheus/prometheus/web.(*Handler).Run
#011/app/web/web.go:575 +0xf08

goroutine 159 [chan receive, 7 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
#011/app/vendor/github.com/soheilhy/cmux/cmux.go:229
google.golang.org/grpc.(*Server).Serve(0xc000026c00, 0x2cd9360, 0xc0000faf60, 0x0, 0x0)
#011/app/vendor/google.golang.org/grpc/server.go:587 +0x22e
github.com/prometheus/prometheus/web.(*Handler).Run.func4(0xc000584e40, 0xc000026c00, 0x2cd9360, 0xc0000faf60)
#011/app/web/web.go:579 +0x3f
created by github.com/prometheus/prometheus/web.(*Handler).Run
#011/app/web/web.go:578 +0xf51

goroutine 160 [IO wait]:
internal/poll.runtime_pollWait(0x7f52353286e0, 0x72, 0x0)
#011/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0005c9b98, 0x72, 0x0, 0x0, 0x251a895)
#011/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
#011/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0005c9b80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
#011/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc0005c9b80, 0x1, 0xc0521be000, 0x100010000)
#011/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0000fae60, 0xc000150a80, 0xc000abeda0, 0x2)
#011/usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc0000fae60, 0x1, 0xc0f947c600, 0x22b71c0, 0x22b71c0)
#011/usr/local/go/src/net/tcpsock.go:261 +0x47
golang.org/x/net/netutil.(*limitListener).Accept(0xc0002819b0, 0xc0f947c5d0, 0xc000281a10, 0x2d0b3e0, 0x438a7e)
#011/app/vendor/golang.org/x/net/netutil/listen.go:48 +0x51
github.com/mwitkow/go-conntrack.(*connTrackListener).Accept(0xc0000faf20, 0x29492c8, 0xc0001a3680, 0x2d0b3e0, 0xc0f947c600)
#011/app/vendor/github.com/mwitkow/go-conntrack/listener_wrapper.go:100 +0x7f
github.com/soheilhy/cmux.(*cMux).Serve(0xc0001a3680, 0x0, 0x0)
#011/app/vendor/github.com/soheilhy/cmux/cmux.go:162 +0xaf
github.com/prometheus/prometheus/web.(*Handler).Run.func5(0xc000584e40, 0x2cf24a0, 0xc0001a3680)
#011/app/web/web.go:582 +0x31
created by github.com/prometheus/prometheus/web.(*Handler).Run
#011/app/web/web.go:581 +0xf8d

goroutine 56 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f5235328610, 0x72, 0xffffffffffffffff)
#011/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0004f6318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
#011/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
#011/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0004f6300, 0xc000aec000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
#011/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0004f6300, 0xc000aec000, 0x8000, 0x8000, 0x45bd50, 0xc000079200, 0x4)
#011/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000e210, 0xc000aec000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
#011/usr/local/go/src/net/net.go:184 +0x68
bufio.(*Reader).Read(0xc000544240, 0xc000168038, 0x9, 0x9, 0x459000, 0xc00051c6c0, 0xc00001adc0)
#011/usr/local/go/src/bufio/bufio.go:226 +0x26a
io.ReadAtLeast(0x2c8f7e0, 0xc000544240, 0xc000168038, 0x9, 0x9, 0x9, 0x100000002239e60, 0xc000522090, 0xc00054a650)
#011/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
#011/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc000168038, 0x9, 0x9, 0x2c8f7e0, 0xc000544240, 0x0, 0xc000000000, 0xc00000dea0, 0xc0005435b0)
#011/app/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000168000, 0xc00075a870, 0x1, 0x0, 0x0)
#011/app/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00013e8c0)
#011/app/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1242 +0x17e
created by google.golang.org/grpc/internal/transport.newHTTP2Client
#011/app/vendor/google.golang.org/grpc/internal/transport/http2_client.go:289 +0xd75

goroutine 57 [select, 7 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000522080, 0x1, 0x0, 0x0, 0x0, 0x0)
#011/app/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x11a
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000544300, 0x0, 0x0)
#011/app/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x1e3
google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00013e8c0)
#011/app/vendor/google.golang.org/grpc/internal/transport/http2_client.go:333 +0x7b
created by google.golang.org/grpc/internal/transport.newHTTP2Client
#011/app/vendor/google.golang.org/grpc/internal/transport/http2_client.go:331 +0xf12

goroutine 161 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f5235328540, 0x72, 0xffffffffffffffff)
#011/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc000524318, 0x72, 0x0, 0x9, 0xffffffffffffffff)
#011/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
#011/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000524300, 0xc0004802d8, 0x9, 0x9, 0x0, 0x0, 0x0)
#011/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc000524300, 0xc0004802d8, 0x9, 0x9, 0x9, 0x0, 0x0)
#011/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0005435a0, 0xc0004802d8, 0x9, 0x9, 0x0, 0x0, 0x0)
#011/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc00020c320, 0xc0004802d8, 0x9, 0x9, 0xc000076e00, 0x7f523776c6d0, 0x0)
#011/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
io.ReadAtLeast(0x2c91480, 0xc00020c320, 0xc0004802d8, 0x9, 0x9, 0x9, 0x912ff8, 0xc00058e0f0, 0xc000580004)
#011/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
#011/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0004802d8, 0x9, 0x9, 0x2c91480, 0xc00020c320, 0x0, 0x0, 0xc00058e0f0, 0x0)
#011/app/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0004802a0, 0x2ca8fe0, 0xc00058e0f0, 0x0, 0x0)
#011/app/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
github.com/soheilhy/cmux.matchHTTP2Field(0x7f5235328d98, 0xc00075a750, 0x2c91480, 0xc00020c320, 0x2523287, 0xc, 0xc00000de20, 0x40a1b5)
#011/app/vendor/github.com/soheilhy/cmux/matchers.go:236 +0x148
github.com/soheilhy/cmux.HTTP2MatchHeaderFieldSendSettings.func1(0x7f5235328d98, 0xc00075a750, 0x2c91480, 0xc00020c320, 0xc00075a750)
#011/app/vendor/github.com/soheilhy/cmux/matchers.go:173 +0xca
github.com/soheilhy/cmux.(*cMux).serve(0xc0001a3680, 0x2d0b3e0, 0xc00075a750, 0xc000150b40, 0xc000645aa0)
#011/app/vendor/github.com/soheilhy/cmux/cmux.go:184 +0x1ee
created by github.com/soheilhy/cmux.(*cMux).Serve
#011/app/vendor/github.com/soheilhy/cmux/cmux.go:171 +0x1a3

goroutine 189 [select, 7 minutes]:
github.com/prometheus/prometheus/tsdb/wal.(*WAL).run(0xc0004230e0)
#011/app/tsdb/wal/wal.go:321 +0xdb
created by github.com/prometheus/prometheus/tsdb/wal.NewSize
#011/app/tsdb/wal/wal.go:290 +0x33f

goroutine 2834 [chan receive]:
github.com/prometheus/prometheus/tsdb.(*Head).processWALSamples(0xc00012e000, 0x16fb9cfc900, 0xc197fe7c80, 0xc197fe7c20, 0x0)
#011/app/tsdb/head.go:302 +0x194
github.com/prometheus/prometheus/tsdb.(*Head).loadWAL.func2(0xc00012e000, 0xc19d144770, 0xc19d144780, 0xc197fe7c80, 0xc197fe7c20)
#011/app/tsdb/head.go:389 +0x48
created by github.com/prometheus/prometheus/tsdb.(*Head).loadWAL
#011/app/tsdb/head.go:388 +0x21d

goroutine 2833 [chan receive]:
github.com/prometheus/prometheus/tsdb.(*Head).processWALSamples(0xc00012e000, 0x16fb9cfc900, 0xc197fe7bc0, 0xc197fe7b60, 0x0)
#011/app/tsdb/head.go:302 +0x194
github.com/prometheus/prometheus/tsdb.(*Head).loadWAL.func2(0xc00012e000, 0xc19d144770, 0xc19d144780, 0xc197fe7bc0, 0xc197fe7b60)
#011/app/tsdb/head.go:389 +0x48
created by github.com/prometheus/prometheus/tsdb.(*Head).loadWAL
#011/app/tsdb/head.go:388 +0x21d

goroutine 2783 [runnable]:
github.com/prometheus/prometheus/tsdb.(*memSeries).append(0xc012c8ecc0, 0x1705e438131, 0x416233c720000000, 0xc1bedc3268)
#011/app/tsdb/head.go:1761 +0x239
github.com/prometheus/prometheus/tsdb.(*Head).processWALSamples(0xc00012e000, 0x16fb9cfc900, 0xc197fe7a40, 0xc197fe79e0, 0x0)
#011/app/tsdb/head.go:316 +0x297
github.com/prometheus/prometheus/tsdb.(*Head).loadWAL.func2(0xc00012e000, 0xc19d144770, 0xc19d144780, 0xc197fe7a40, 0xc197fe79e0)
#011/app/tsdb/head.go:389 +0x48
created by github.com/prometheus/prometheus/tsdb.(*Head).loadWAL
#011/app/tsdb/head.go:388 +0x21d

goroutine 2784 [running]:
#011goroutine running on other thread; stack unavailable
created by github.com/prometheus/prometheus/tsdb.(*Head).loadWAL
#011/app/tsdb/head.go:388 +0x21d