gistfile1.txt

"=== received SIGQUIT ===\n*** goroutine dump...\ngoroutine 1 [running]:\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/signals.SignalHandlerLoop(0x7fad603dc168, 0xc4200106e0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/signals/signals.go:37 +0x35c\ngithub.com/weaveworks/cortex/server.(*Server).Run(0xc4202c2690)\n\t/go/src/github.com/weaveworks/cortex/server/server.go:110 +0x474\nmain.main()\n\t/go/src/github.com/weaveworks/cortex/cmd/ingester/main.go:52 +0x473\n\ngoroutine 17 [syscall, 13 minutes, locked to thread]:\nruntime.goexit()\n\t/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1\n\ngoroutine 57 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016bb50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016bb40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bb00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x3)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 56 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016bad0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016bac0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016ba80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 21 [syscall]:\nos/signal.signal_recv(0x1036040)\n\t/usr/local/go/src/runtime/sigqueue.go:116 +0x157\nos/signal.loop()\n\t/usr/local/go/src/os/signal/signal_unix.go:22 +0x22\ncreated by os/signal.init.1\n\t/usr/local/go/src/os/signal/signal_unix.go:28 +0x41\n\ngoroutine 23 [select]:\nnet/http.(*persistConn).roundTrip(0xc438274a00, 0xc43e614ca0, 0x0, 0x0, 0x0)\n\t/usr/local/go/src/net/http/transport.go:1840 +0x93b\nnet/http.(*Transport).RoundTrip(0xc420286000, 0xc447466d20, 0xc420286000, 0x0, 0x0)\n\t/usr/local/go/src/net/http/transport.go:380 +0x4ee\nnet/http.send(0xc447466d20, 0x1030bc0, 0xc420286000, 0x0, 0x0, 0x0, 0x8, 0xc43d2ffa40, 0xc438377e40)\n\t/usr/local/go/src/net/http/client.go:256 +0x15f\nnet/http.(*Client).send(0xc4201b1590, 0xc447466d20, 0x0, 0x0, 0x0, 0xc438377e40, 0x0, 0x1)\n\t/usr/local/go/src/net/http/client.go:146 +0x102\nnet/http.(*Client).doFollowingRedirects(0xc4201b1590, 0xc447466d20, 0xb47ac8, 0x3, 0x588a5a01, 0x106792f8)\n\t/usr/local/go/src/net/http/client.go:528 +0x5e5\nnet/http.(*Client).Do(0xc4201b1590, 0xc447466d20, 0x1050960, 0x0, 0x1)\n\t/usr/local/go/src/net/http/client.go:184 +0x1ea\ngithub.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api.(*Client).doRequest(0xc4201865a0, 0xc4395d7900, 0xc43d2ffd30, 0xc43e614c20, 0x16, 0xc4395d7900)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api/api.go:475 +0x8e\ngithub.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api.(*KV).getInternal(0xc42017a1c8, 0xc42019a730, 0xf, 0x0, 0xc4395d78b0, 0xc420180000, 0xc43d2ffe00, 0x410558, 0x50)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api/kv.go:141 +0x233\ngithub.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api.(*KV).Get(0xc42017a1c8, 0xc42019a730, 0xf, 0xc4395d78b0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api/kv.go:75 +0x8c\ngithub.com/weaveworks/cortex/ring.(*consulClient).WatchKey(0xc4201792e0, 0xc42019a730, 0xf, 0xc420186600, 0xc42019a720)\n\t/go/src/github.com/weaveworks/cortex/ring/consul_client.go:292 +0x117\ngithub.com/weaveworks/cortex/ring.(*prefixedConsulClient).WatchKey(0xc420179300, 0xaf3adc, 0x4, 0xc420186600, 0xc42019a720)\n\t/go/src/github.com/weaveworks/cortex/ring/consul_client.go:353 +0xa6\ngithub.com/weaveworks/cortex/ring.(*Ring).loop(0xc4201866c0)\n\t/go/src/github.com/weaveworks/cortex/ring/ring.go:119 +0xbe\ncreated by github.com/weaveworks/cortex/ring.New\n\t/go/src/github.com/weaveworks/cortex/ring/ring.go:96 +0x38e\n\ngoroutine 24 [select]:\ngithub.com/weaveworks/cortex/ring.(*IngesterRegistration).heartbeat(0xc4202444d0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ring/ingester_lifecycle.go:224 +0x3bc\ngithub.com/weaveworks/cortex/ring.(*IngesterRegistration).loop(0xc4202444d0)\n\t/go/src/github.com/weaveworks/cortex/ring/ingester_lifecycle.go:153 +0xb9\ncreated by github.com/weaveworks/cortex/ring.RegisterIngester\n\t/go/src/github.com/weaveworks/cortex/ring/ingester_lifecycle.go:123 +0x31f\n\ngoroutine 55 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016ba50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016ba40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016ba00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 77 [select]:\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc42000c300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855\n\ngoroutine 1283 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204962c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204962c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d5a7cd0, 0x2, 0x2, 0xc4293e5448, 0x0, 0xc42d8d94c0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d5b53e0, 0x159dc604330, 0x159dc64d710, 0xc42d5a7cd0, 0x2, 0x2, 0xc42d5b5300, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d5b53e0, 0xc42d5b52f0, 0x0, 0xc42d5a7ca8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d5b53e0, 0xa934c0, 0xc42d5b52f0, 0x7fad6035c8e0, 0xc42d5b53e0, 0x1038500, 0xc42d5b53b0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d5b53b0, 0xa934c0, 0xc42d5b52f0, 0xc42d946d40, 0xc42d946d60, 0x7fad6035c8e0, 0xc42d5b53b0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d5b53b0, 0xa934c0, 0xc42d5b52f0, 0x7fad6035c8e0, 0xc42d5b53b0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d5b53b0, 0xa934c0, 0xc42d5b52f0, 0xc42d946d40, 0xc42d946d80, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xffffffff, 0x0, 0x19, 0xc4293e57e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xc42d946d40, 0xc42d946da0, 0xc4293e5878, 0x558012, 0x588a5743, 0x26123b93)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xed01c4e43, 0x0, 0x0, 0xcea1)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xc42d946d40, 0xc42d946dc0, 0xa91b40, 0xc42d946de0, 0xc42d946d40, 0xc42d946de0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xc420246100, 0xc42d5a7c60, 0xc4293e5a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xc42d946d40, 0xc42d946d60, 0x4e, 0xc420314b18, 0x0, 0xc420314b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d5b51d0, 0xc42d5aa420, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d054e00, 0xc420331380, 0x10265d8, 0xc42d5b5290, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d054e00, 0xc42d5b5290)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d054e00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 1015 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc420497484)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc420497480)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0xd6f76da883a32dd9)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a3e2720, 0xc420162418, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a3cbef0, 0xc42a3e2720, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a3cbef0, 0xc423f4dbc0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a3cbef0, 0xc4240da3e0, 0x0, 0xc42a3de1d8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a3cbef0, 0xa74a80, 0xc4240da3e0, 0x7fad6035c8e0, 0xc42a3cbef0, 0x1038500, 0xc42a3cbec0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a3cbec0, 0xa74a80, 0xc4240da3e0, 0xc423f4cba0, 0xc423f4cbc0, 0x7fad6035c8e0, 0xc42a3cbec0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a3cbec0, 0xa74a80, 0xc4240da3e0, 0x7fad6035c8e0, 0xc42a3cbec0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a3cbec0, 0xa74a80, 0xc4240da3e0, 0xc423f4cba0, 0xc423f4cc00, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xffffffff, 0x0, 0x19, 0xc42910b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xc423f4cba0, 0xc423f4cc40, 0xc42910b878, 0x558012, 0x588a5741, 0x3904a859)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xed01c4e41, 0x0, 0x0, 0x1646d8)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xc423f4cba0, 0xc423f4cc80, 0xa91b40, 0xc423f4cd40, 0xc423f4cba0, 0xc423f4cd40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xc420246100, 0xc42a33c9d0, 0xc42910ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xc423f4cba0, 0xc423f4cbc0, 0xaa6b, 0xc420044b18, 0x0, 0xc420044b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a187f50, 0xc42a1ffaa0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204ff600, 0xc420331380, 0x10265c0, 0xc42a187f80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204ff600, 0xc42a187f80)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204ff600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 76 [IO wait]:\nnet.runtime_pollWait(0x7fad6039d120, 0x72, 0x4)\n\t/usr/local/go/src/runtime/netpoll.go:160 +0x59\nnet.(*pollDesc).wait(0xc42018c3e0, 0x72, 0xc42004b8d8, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38\nnet.(*pollDesc).waitRead(0xc42018c3e0, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34\nnet.(*netFD).Read(0xc42018c380, 0xc4200ee000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_unix.go:243 +0x1a1\nnet.(*conn).Read(0xc42017a170, 0xc4200ee000, 0x8000, 0x8000, 0x0, 0x0, 0x0)\n\t/usr/local/go/src/net/net.go:173 +0x70\nbufio.(*Reader).fill(0xc4201860c0)\n\t/usr/local/go/src/bufio/bufio.go:97 +0x10c\nbufio.(*Reader).Read(0xc4201860c0, 0xc42000c278, 0x9, 0x9, 0x43fe80, 0xc43d04cb60, 0xc4334ef590)\n\t/usr/local/go/src/bufio/bufio.go:209 +0x1bc\nio.ReadAtLeast(0x102f380, 0xc4201860c0, 0xc42000c278, 0x9, 0x9, 0x9, 0xc42004bae8, 0x451a90, 0xc447e8e1c0)\n\t/usr/local/go/src/io/io.go:307 +0xa4\nio.ReadFull(0x102f380, 0xc4201860c0, 0xc42000c278, 0x9, 0x9, 0xc43d04cb60, 0x1, 0x2)\n\t/usr/local/go/src/io/io.go:325 +0x58\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc42000c278, 0x9, 0x9, 0x102f380, 0xc4201860c0, 0x0, 0xc400000000, 0xc42004bc80, 0x7f3105)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42000c240, 0x399b, 0x399b, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420199320, 0xc432f42c00, 0xc432f42c00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42000c300, 0xc420199680, 0xb475b8)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc42000c300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc42017a170, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc42017a170)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac\n\ngoroutine 52 [select]:\ngithub.com/weaveworks/cortex/chunk.(*MemcacheClient).updateLoop(0xc420171630, 0xdf8475800, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/chunk/memcache_client.go:78 +0x1fe\ncreated by github.com/weaveworks/cortex/chunk.NewMemcacheClient\n\t/go/src/github.com/weaveworks/cortex/chunk/memcache_client.go:63 +0x24a\n\ngoroutine 54 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016b9d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016b9c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016b980, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 10433 [semacquire, 6 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4384c1da0, 0xc4384c1e00, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4384c1da0, 0xc4488dea20, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4384c1da0, 0xc445991f80, 0x0, 0xc4391454a0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4384c1da0, 0xa74a80, 0xc445991f80, 0x7fad6035c8e0, 0xc4384c1da0, 0x1038500, 0xc4384c1d40)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4384c1d40, 0xa74a80, 0xc445991f80, 0xc43b967700, 0xc43b967820, 0x7fad6035c8e0, 0xc4384c1d40, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4384c1d40, 0xa74a80, 0xc445991f80, 0x7fad6035c8e0, 0xc4384c1d40, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4384c1d40, 0xa74a80, 0xc445991f80, 0xc43b967700, 0xc43b9679a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0xffffffff, 0x0, 0x19, 0xc432f7b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0xc43b967700, 0xc43b967b20, 0xc432f7b878, 0x558012, 0x588a58e2, 0xfb84a51)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0x41167e, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0xc43b967700, 0xc43b967c80, 0xa91b40, 0xc43b967dc0, 0xc43b967700, 0xc43b967dc0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0xc420246100, 0xc44538ace0, 0xc432f7ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0xc43b967700, 0xc43b967820, 0x61d2, 0xc440d67b18, 0x0, 0xc440d67b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44abb34d0, 0xc44ac1ed80, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ac00, 0xc420331380, 0x10265c0, 0xc44ac279e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ac00, 0xc44ac279e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ac00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 58 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016bbd0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016bbc0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bb80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x4)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 59 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016bc50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016bc40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bc00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x5)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 60 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016bcd0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016bcc0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bc80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x6)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 61 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016bd50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016bd40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bd00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x7)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 62 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016bdd0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016bdc0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bd80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x8)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 63 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016be50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016be40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016be00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x9)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 64 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016bed0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016bec0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016be80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xa)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 65 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016bf50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016bf40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bf00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xb)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 114 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42016bfd0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42016bfc0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bf80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xc)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 115 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c050, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c040)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c000, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xd)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 116 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c0d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c0c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c080, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xe)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 117 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c150, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c140)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c100, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xf)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 118 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c1d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c1c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c180, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x10)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 119 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c250, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c240)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c200, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x11)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 120 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c2d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c2c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c280, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x12)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 121 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c350, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c340)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c300, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x13)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 122 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c3d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c3c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c380, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x14)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 123 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c450, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c440)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c400, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x15)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 124 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c4d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c4c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c480, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x16)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 125 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c550, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c540)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c500, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x17)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 126 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c5d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c5c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c580, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x18)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 127 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c650, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c640)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c600, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x19)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 128 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c6d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c6c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c680, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1a)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 129 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c750, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c740)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c700, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1b)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 130 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c7d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c7c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c780, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1c)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 131 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c850, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c840)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c800, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1d)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 132 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c8d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c8c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c880, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1e)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 133 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c950, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c940)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c900, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1f)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 134 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034c9d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034c9c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c980, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x20)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 135 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034ca50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034ca40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034ca00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x21)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 136 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034cad0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034cac0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034ca80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x22)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 137 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034cb50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034cb40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cb00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x23)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 138 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034cbd0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034cbc0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cb80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x24)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 139 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034cc50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034cc40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cc00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x25)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 140 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034ccd0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034ccc0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cc80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x26)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 141 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034cd50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034cd40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cd00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x27)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 142 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034cdd0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034cdc0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cd80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x28)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 143 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034ce50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034ce40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034ce00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x29)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 144 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034ced0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034cec0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034ce80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2a)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 145 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034cf50, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034cf40)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cf00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2b)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 146 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034cfd0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034cfc0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cf80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2c)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 147 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034d050, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034d040)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034d000, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2d)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 148 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034d0d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034d0c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034d080, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2e)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 149 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034d150, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034d140)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034d100, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2f)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 150 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034d1d0, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034d1c0)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034d180, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x30)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 151 [semacquire, 13 minutes]:\nsync.runtime_notifyListWait(0xc42034d250, 0x0)\n\t/usr/local/go/src/runtime/sema.go:267 +0x122\nsync.(*Cond).Wait(0xc42034d240)\n\t/usr/local/go/src/sync/cond.go:57 +0x80\ngithub.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034d200, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94\ngithub.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x31)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9\n\ngoroutine 152 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3080)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).Lock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:91 +0x98\ngithub.com/weaveworks/cortex/ingester.(*userStates).cp(0xc4201e3078, 0xc420316f8e)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:39 +0x42\ngithub.com/weaveworks/cortex/ingester.(*Ingester).sweepUsers(0xc4201e3040, 0xc4201e4f00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:404 +0x59\ngithub.com/weaveworks/cortex/ingester.(*Ingester).loop(0xc4201e3040)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:389 +0x134\ncreated by github.com/weaveworks/cortex/ingester.New\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:187 +0xa17\n\ngoroutine 153 [IO wait]:\nnet.runtime_pollWait(0x7fad6039d1e0, 0x72, 0x0)\n\t/usr/local/go/src/runtime/netpoll.go:160 +0x59\nnet.(*pollDesc).wait(0xc42010a7d0, 0x72, 0xc42004ad28, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38\nnet.(*pollDesc).waitRead(0xc42010a7d0, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34\nnet.(*netFD).accept(0xc42010a770, 0x0, 0x10312c0, 0xc42a914800)\n\t/usr/local/go/src/net/fd_unix.go:419 +0x238\nnet.(*TCPListener).accept(0xc420038210, 0x29e8d60800, 0x0, 0x0)\n\t/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e\nnet.(*TCPListener).AcceptTCP(0xc420038210, 0xc42004ae50, 0xc42004ae58, 0xc42004ae48)\n\t/usr/local/go/src/net/tcpsock.go:209 +0x49\nnet/http.tcpKeepAliveListener.Accept(0xc420038210, 0xb47918, 0xc422c48100, 0x1038500, 0xc4201816b0)\n\t/usr/local/go/src/net/http/server.go:2608 +0x2f\nnet/http.(*Server).Serve(0xc42007ae80, 0x1037e80, 0xc420038210, 0x0, 0x0)\n\t/usr/local/go/src/net/http/server.go:2273 +0x1ce\nnet/http.(*Server).ListenAndServe(0xc42007ae80, 0xc42007ae80, 0x0)\n\t/usr/local/go/src/net/http/server.go:2219 +0xb4\nnet/http.ListenAndServe(0xc4203401c8, 0x3, 0x1032a80, 0xc4203315c0, 0x3, 0x10)\n\t/usr/local/go/src/net/http/server.go:2351 +0xa0\ncreated by github.com/weaveworks/cortex/server.(*Server).Run\n\t/go/src/github.com/weaveworks/cortex/server/server.go:100 +0x29f\n\ngoroutine 154 [IO wait, 13 minutes]:\nnet.runtime_pollWait(0x7fad6039d060, 0x72, 0x0)\n\t/usr/local/go/src/runtime/netpoll.go:160 +0x59\nnet.(*pollDesc).wait(0xc420242f40, 0x72, 0xc42023fd70, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38\nnet.(*pollDesc).waitRead(0xc420242f40, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34\nnet.(*netFD).accept(0xc420242ee0, 0x0, 0x10312c0, 0xc43e0cd620)\n\t/usr/local/go/src/net/fd_unix.go:419 +0x238\nnet.(*TCPListener).accept(0xc42016e1b8, 0x43356e, 0xc42023fe20, 0x63e1dc)\n\t/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e\nnet.(*TCPListener).Accept(0xc42016e1b8, 0xb47580, 0xc4202460c0, 0x103ae60, 0xc43a4a07d8)\n\t/usr/local/go/src/net/tcpsock.go:222 +0x49\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve(0xc4202460c0, 0x1037580, 0xc42016e1b8, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:368 +0x187\ncreated by github.com/weaveworks/cortex/server.(*Server).Run\n\t/go/src/github.com/weaveworks/cortex/server/server.go:108 +0x419\n\ngoroutine 155 [select, 13 minutes, locked to thread]:\nruntime.gopark(0xb47f10, 0x0, 0xaf59e3, 0x6, 0x18, 0x2)\n\t/usr/local/go/src/runtime/proc.go:259 +0x13a\nruntime.selectgoImpl(0xc42036af30, 0x0, 0x18)\n\t/usr/local/go/src/runtime/select.go:423 +0x11d9\nruntime.selectgo(0xc42036af30)\n\t/usr/local/go/src/runtime/select.go:238 +0x1c\nruntime.ensureSigM.func1()\n\t/usr/local/go/src/runtime/signal1_unix.go:304 +0x2f3\nruntime.goexit()\n\t/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1\n\ngoroutine 8355 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42c7ac4e0, 0xc42c7ac510, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42c7ac4e0, 0xc43a3ea9e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42c7ac4e0, 0xc43ddc5ba0, 0x0, 0xc42c7ab458, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42c7ac4e0, 0xa74a80, 0xc43ddc5ba0, 0x7fad6035c8e0, 0xc42c7ac4e0, 0x1038500, 0xc42c7ac4b0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c7ac4b0, 0xa74a80, 0xc43ddc5ba0, 0xc43a3ea8e0, 0xc43a3ea900, 0x7fad6035c8e0, 0xc42c7ac4b0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c7ac4b0, 0xa74a80, 0xc43ddc5ba0, 0x7fad6035c8e0, 0xc42c7ac4b0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c7ac4b0, 0xa74a80, 0xc43ddc5ba0, 0xc43a3ea8e0, 0xc43a3ea920, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xffffffff, 0x0, 0x19, 0xc43a6677e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xc43a3ea8e0, 0xc43a3ea940, 0xc43a667878, 0x558012, 0x588a579b, 0x39a1dd86)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xed01c4e9b, 0x0, 0x0, 0xad19c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xc43a3ea8e0, 0xc43a3ea960, 0xa91b40, 0xc43a3ea980, 0xc43a3ea8e0, 0xc43a3ea980)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xc420246100, 0xc42cb083a0, 0xc43a667a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xc43a3ea8e0, 0xc43a3ea900, 0x8755, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42cbe5cb0, 0xc42cbf63c0, 0xc420179d60, 0x0, 0x0, 0x637573070a0f0a63, 0x7274041273736563)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad700, 0xc420331380, 0x10265c0, 0xc42cbe5ce0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad700, 0xc42cbe5ce0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad700)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 12099 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc440ce2f00, 0xc440ce2f30, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc440ce2f00, 0xc4437ad520, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc440ce2f00, 0xc448583840, 0x0, 0xc42b08ced8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc440ce2f00, 0xa74a80, 0xc448583840, 0x7fad6035c8e0, 0xc440ce2f00, 0x1038500, 0xc440ce2ed0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc440ce2ed0, 0xa74a80, 0xc448583840, 0xc4437ad3e0, 0xc4437ad400, 0x7fad6035c8e0, 0xc440ce2ed0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440ce2ed0, 0xa74a80, 0xc448583840, 0x7fad6035c8e0, 0xc440ce2ed0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc440ce2ed0, 0xa74a80, 0xc448583840, 0xc4437ad3e0, 0xc4437ad440, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0xffffffff, 0x0, 0x19, 0xc4371bb7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0xc4437ad3e0, 0xc4437ad460, 0xc4371bb878, 0x558012, 0x588a59f0, 0x102a6306)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0x41167e, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0xc4437ad3e0, 0xc4437ad480, 0xa91b40, 0xc4437ad4a0, 0xc4437ad3e0, 0xc4437ad4a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0xc420246100, 0xc42c15d720, 0xc4371bba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0xc4437ad3e0, 0xc4437ad400, 0x7f2c, 0xc420313b18, 0x0, 0xc420313b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44809f890, 0xc43dc3de60, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc43859e400, 0xc420331380, 0x10265c0, 0xc44809f8c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc43859e400, 0xc44809f8c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc43859e400)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8919 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43756d4d0, 0xc43756d500, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43756d4d0, 0xc442132180, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43756d4d0, 0xc44623ef60, 0x0, 0xc42b6f1f08, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43756d4d0, 0xa74a80, 0xc44623ef60, 0x7fad6035c8e0, 0xc43756d4d0, 0x1038500, 0xc43756d4a0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43756d4a0, 0xa74a80, 0xc44623ef60, 0xc442132080, 0xc4421320a0, 0x7fad6035c8e0, 0xc43756d4a0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43756d4a0, 0xa74a80, 0xc44623ef60, 0x7fad6035c8e0, 0xc43756d4a0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43756d4a0, 0xa74a80, 0xc44623ef60, 0xc442132080, 0xc4421320c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xffffffff, 0x0, 0x19, 0xc44a87b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xc442132080, 0xc4421320e0, 0xc44a87b878, 0x558012, 0x588a57f2, 0xf0c3774)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xed01c4ef2, 0x0, 0x0, 0x3fcd45)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xc442132080, 0xc442132100, 0xa91b40, 0xc442132120, 0xc442132080, 0xc442132120)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xc420246100, 0xc4278f6360, 0xc44a87ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xc442132080, 0xc4421320a0, 0x4f3f, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442347680, 0xc42a5f5380, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5986c5)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ad00, 0xc420331380, 0x10265c0, 0xc4423476b0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ad00, 0xc4423476b0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ad00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8736 [chan receive, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc43abf3ae0, 0xc44776e870)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47\nnet/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc43abf3ae0, 0xc44776e870)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc43abf3ae0, 0xc44776e870)\n\t/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc4324df4c0, 0xc44776e870)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab\nnet/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc4324df4c0, 0xc44776e870)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc4324df4c0, 0xc44776e870)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc4324df4a0, 0xc44776e690)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2\nnet/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc4324df4a0, 0xc44776e690)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc4324df480, 0xc44776e690)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c\nnet/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc4324df480, 0xc44776e690)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4252bde10, 0xc44776e690)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449\nnet/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4252bde10, 0xc44776e5a0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4252bde10, 0xc44776e5a0)\n\t/usr/local/go/src/net/http/server.go:2202 +0x7d\nnet/http.(*conn).serve(0xc43aeff500, 0x1038440, 0xc423686fc0)\n\t/usr/local/go/src/net/http/server.go:1579 +0x4b7\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:2293 +0x44d\n\ngoroutine 5313 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048c504)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048c500)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x2dce913412292e09)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc446b45da0, 0xc422334c2d, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc446b458c0, 0xc446b45da0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc446b458c0, 0xc42b71b300, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc446b458c0, 0xc42ba151c0, 0x0, 0xc446a05338, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc446b458c0, 0xa74a80, 0xc42ba151c0, 0x7fad6035c8e0, 0xc446b458c0, 0x1038500, 0xc446b45890)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc446b45890, 0xa74a80, 0xc42ba151c0, 0xc42b71ac00, 0xc42b71ac20, 0x7fad6035c8e0, 0xc446b45890, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc446b45890, 0xa74a80, 0xc42ba151c0, 0x7fad6035c8e0, 0xc446b45890, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc446b45890, 0xa74a80, 0xc42ba151c0, 0xc42b71ac00, 0xc42b71ac60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xffffffff, 0x0, 0x19, 0xc42d70d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xc42b71ac00, 0xc42b71ac80, 0xc42d70d878, 0x558012, 0x588a575f, 0x344a53de)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xed01c4e5f, 0x0, 0x0, 0x173b0c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xc42b71ac00, 0xc42b71aca0, 0xa91b40, 0xc42b71ace0, 0xc42b71ac00, 0xc42b71ace0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xc420246100, 0xc4494205e0, 0xc42d70da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xc42b71ac00, 0xc42b71ac20, 0x8af4, 0xc440d67b18, 0x0, 0xc440d67b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc447b11e00, 0xc43c5cbb00, 0xc420179d60, 0x0, 0x0, 0xc42ad71fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cbe300, 0xc420331380, 0x10265c0, 0xc447b11e90, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cbe300, 0xc447b11e90)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cbe300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8321 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a56bd40, 0xc42a56bd70, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a56bd40, 0xc437d05d80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a56bd40, 0xc437d59b60, 0x0, 0xc42a6869d8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a56bd40, 0xa74a80, 0xc437d59b60, 0x7fad6035c8e0, 0xc42a56bd40, 0x1038500, 0xc42a56bd10)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a56bd10, 0xa74a80, 0xc437d59b60, 0xc437d05c40, 0xc437d05c60, 0x7fad6035c8e0, 0xc42a56bd10, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a56bd10, 0xa74a80, 0xc437d59b60, 0x7fad6035c8e0, 0xc42a56bd10, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a56bd10, 0xa74a80, 0xc437d59b60, 0xc437d05c40, 0xc437d05c80, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xffffffff, 0x0, 0x19, 0xc4464b77e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xc437d05c40, 0xc437d05ca0, 0xc4464b7878, 0x558012, 0x588a5798, 0xdb38f7a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xed01c4e98, 0x0, 0x0, 0x648265)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xc437d05c40, 0xc437d05d00, 0xa91b40, 0xc437d05d20, 0xc437d05c40, 0xc437d05d20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xc420246100, 0xc42b9ddd60, 0xc4464b7a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xc437d05c40, 0xc437d05c60, 0x5d64, 0xc420046b18, 0x0, 0xc420046b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42f4074a0, 0xc4296fa3c0, 0xc420179d60, 0x0, 0x0, 0xc4202e17a8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67f00, 0xc420331380, 0x10265c0, 0xc42f4074d0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67f00, 0xc42f4074d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67f00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8322 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4288a6d20, 0xc4288a6d50, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4288a6d20, 0xc437da04c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4288a6d20, 0xc437e1f420, 0x0, 0xc42c8d46d0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4288a6d20, 0xa74a80, 0xc437e1f420, 0x7fad6035c8e0, 0xc4288a6d20, 0x1038500, 0xc4288a6cf0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4288a6cf0, 0xa74a80, 0xc437e1f420, 0xc437da03a0, 0xc437da03c0, 0x7fad6035c8e0, 0xc4288a6cf0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4288a6cf0, 0xa74a80, 0xc437e1f420, 0x7fad6035c8e0, 0xc4288a6cf0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4288a6cf0, 0xa74a80, 0xc437e1f420, 0xc437da03a0, 0xc437da03e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xffffffff, 0x0, 0x19, 0xc43b4f17e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xc437da03a0, 0xc437da0400, 0xc43b4f1878, 0x558012, 0x588a5798, 0xda24519)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xed01c4e98, 0x0, 0x0, 0x1698ea)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xc437da03a0, 0xc437da0420, 0xa91b40, 0xc437da0440, 0xc437da03a0, 0xc437da0440)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xc420246100, 0xc426fafea0, 0xc43b4f1a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xc437da03a0, 0xc437da03c0, 0x4351, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42f407740, 0xc44363f800, 0xc420179d60, 0x0, 0x0, 0xc4202e17a8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c2d00, 0xc420331380, 0x10265c0, 0xc42f407770, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c2d00, 0xc42f407770)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c2d00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8734 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc448c247b0, 0xc448c247e0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc448c247b0, 0xc43287f920, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc448c247b0, 0xc43333ff80, 0x0, 0xc448761bf0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc448c247b0, 0xa74a80, 0xc43333ff80, 0x7fad6035c8e0, 0xc448c247b0, 0x1038500, 0xc448c24780)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc448c24780, 0xa74a80, 0xc43333ff80, 0xc43287f800, 0xc43287f820, 0x7fad6035c8e0, 0xc448c24780, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc448c24780, 0xa74a80, 0xc43333ff80, 0x7fad6035c8e0, 0xc448c24780, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc448c24780, 0xa74a80, 0xc43333ff80, 0xc43287f800, 0xc43287f840, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xffffffff, 0x0, 0x19, 0xc42f3897e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xc43287f800, 0xc43287f860, 0xc42f389878, 0x558012, 0x588a57d4, 0xe8eaebe)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xed01c4ed4, 0x0, 0x0, 0x2ec2bf)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xc43287f800, 0xc43287f880, 0xa91b40, 0xc43287f8a0, 0xc43287f800, 0xc43287f8a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xc420246100, 0xc438653d90, 0xc42f389a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xc43287f800, 0xc43287f820, 0x5241, 0xc420e1ab18, 0x0, 0xc420e1ab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43fc60c90, 0xc44993a0c0, 0xc420179d60, 0x0, 0x0, 0xc43bd367a8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421000d00, 0xc420331380, 0x10265c0, 0xc43fc60cc0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421000d00, 0xc43fc60cc0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421000d00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 1119 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204962c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204962c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42b839760, 0x2, 0x2, 0xc4260f7448, 0x0, 0xc42b842380)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42b850a80, 0x159dc604330, 0x159dc64d710, 0xc42b839760, 0x2, 0x2, 0xc42b850a00, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42b850a80, 0xc42b850990, 0x0, 0xc42b839738, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42b850a80, 0xa934c0, 0xc42b850990, 0x7fad6035c8e0, 0xc42b850a80, 0x1038500, 0xc42b850a50)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42b850a50, 0xa934c0, 0xc42b850990, 0xc42b854740, 0xc42b854760, 0x7fad6035c8e0, 0xc42b850a50, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b850a50, 0xa934c0, 0xc42b850990, 0x7fad6035c8e0, 0xc42b850a50, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42b850a50, 0xa934c0, 0xc42b850990, 0xc42b854740, 0xc42b854780, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xffffffff, 0x0, 0x19, 0xc4260f77e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xc42b854740, 0xc42b8547a0, 0xc4260f7878, 0x558012, 0x588a5742, 0x213cc5e5)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xc4260f7900, 0x0, 0x0, 0x3f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xc42b854740, 0xc42b8547c0, 0xa91b40, 0xc42b8547e0, 0xc42b854740, 0xc42b8547e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xc420246100, 0xc42b8396f0, 0xc4260f7a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xc42b854740, 0xc42b854760, 0x4e, 0xc420047b18, 0x0, 0xc420047b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42b850870, 0xc42b5f0660, 0xc420179d60, 0x0, 0x0, 0x2d65646f6e2d6d6f, 0x726574726f707865)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429907200, 0xc420331380, 0x10265d8, 0xc42b850930, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429907200, 0xc42b850930)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429907200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 5574 [IO wait, 6 minutes]:\nnet.runtime_pollWait(0x7fad6039cd60, 0x72, 0x3)\n\t/usr/local/go/src/runtime/netpoll.go:160 +0x59\nnet.(*pollDesc).wait(0xc43d2bf5d0, 0x72, 0xc440d668d8, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38\nnet.(*pollDesc).waitRead(0xc43d2bf5d0, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34\nnet.(*netFD).Read(0xc43d2bf570, 0xc43c8de000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_unix.go:243 +0x1a1\nnet.(*conn).Read(0xc4464e30c0, 0xc43c8de000, 0x8000, 0x8000, 0x0, 0x0, 0x0)\n\t/usr/local/go/src/net/net.go:173 +0x70\nbufio.(*Reader).fill(0xc445d82fc0)\n\t/usr/local/go/src/bufio/bufio.go:97 +0x10c\nbufio.(*Reader).Read(0xc445d82fc0, 0xc42000d6b8, 0x9, 0x9, 0x43fe80, 0xc4351c4240, 0x0)\n\t/usr/local/go/src/bufio/bufio.go:209 +0x1bc\nio.ReadAtLeast(0x102f380, 0xc445d82fc0, 0xc42000d6b8, 0x9, 0x9, 0x9, 0xc440d66ae8, 0x451a90, 0xc42b28bd00)\n\t/usr/local/go/src/io/io.go:307 +0xa4\nio.ReadFull(0x102f380, 0xc445d82fc0, 0xc42000d6b8, 0x9, 0x9, 0xc4351c4240, 0x0, 0x1)\n\t/usr/local/go/src/io/io.go:325 +0x58\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc42000d6b8, 0x9, 0x9, 0x102f380, 0xc445d82fc0, 0x0, 0xc400000000, 0xc440d66c80, 0x7f3105)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42000d680, 0xf, 0xf, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4408cc8d0, 0xc42f4308a0, 0xc42f4308a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42000d740, 0xc4408cc960, 0xb475b8)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc42000d740)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc4464e30c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc4464e30c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac\n\ngoroutine 870 [IO wait, 13 minutes]:\nnet.runtime_pollWait(0x7fad6039cee0, 0x72, 0x9)\n\t/usr/local/go/src/runtime/netpoll.go:160 +0x59\nnet.(*pollDesc).wait(0xc427bb3b80, 0x72, 0xc4203128d8, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38\nnet.(*pollDesc).waitRead(0xc427bb3b80, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34\nnet.(*netFD).Read(0xc427bb3b20, 0xc4293a6000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_unix.go:243 +0x1a1\nnet.(*conn).Read(0xc428d0b6f8, 0xc4293a6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)\n\t/usr/local/go/src/net/net.go:173 +0x70\nbufio.(*Reader).fill(0xc4287d9d40)\n\t/usr/local/go/src/bufio/bufio.go:97 +0x10c\nbufio.(*Reader).Read(0xc4287d9d40, 0xc426048278, 0x9, 0x9, 0x43fe80, 0xc42de31f20, 0x0)\n\t/usr/local/go/src/bufio/bufio.go:209 +0x1bc\nio.ReadAtLeast(0x102f380, 0xc4287d9d40, 0xc426048278, 0x9, 0x9, 0x9, 0xc420312ae8, 0x451a90, 0xc421266a80)\n\t/usr/local/go/src/io/io.go:307 +0xa4\nio.ReadFull(0x102f380, 0xc4287d9d40, 0xc426048278, 0x9, 0x9, 0xc42de31f20, 0x0, 0x1)\n\t/usr/local/go/src/io/io.go:325 +0x58\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc426048278, 0x9, 0x9, 0x102f380, 0xc4287d9d40, 0x0, 0xc400000000, 0xc420312c80, 0x7f3105)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc426048240, 0x5a, 0x5a, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc429291620, 0xc42de39c80, 0xc42de39c80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc426048300, 0xc4292916b0, 0xb475b8)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc426048300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc428d0b6f8, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc428d0b6f8)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac\n\ngoroutine 11264 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42516a930, 0xc42516a960, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42516a930, 0xc43c693ee0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42516a930, 0xc4438a9a60, 0x0, 0xc4320da8c0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42516a930, 0xa74a80, 0xc4438a9a60, 0x7fad6035c8e0, 0xc42516a930, 0x1038500, 0xc42516a900)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42516a900, 0xa74a80, 0xc4438a9a60, 0xc43c693de0, 0xc43c693e00, 0x7fad6035c8e0, 0xc42516a900, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42516a900, 0xa74a80, 0xc4438a9a60, 0x7fad6035c8e0, 0xc42516a900, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42516a900, 0xa74a80, 0xc4438a9a60, 0xc43c693de0, 0xc43c693e20, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xffffffff, 0x0, 0x19, 0xc43618b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xc43c693de0, 0xc43c693e40, 0xc43618b878, 0x558012, 0x588a597b, 0x39877a42)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xed01c507b, 0x0, 0x0, 0xe3173)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xc43c693de0, 0xc43c693e60, 0xa91b40, 0xc43c693e80, 0xc43c693de0, 0xc43c693e80)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xc420246100, 0xc4322de280, 0xc43618ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xc43c693de0, 0xc43c693e00, 0x6ec4, 0xc440d67b18, 0x0, 0xc440d67b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4243633b0, 0xc44346cf60, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f62ebe6)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc43859ec00, 0xc420331380, 0x10265c0, 0xc4243633e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc43859ec00, 0xc4243633e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc43859ec00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8981 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4424c1f50, 0xc4424c1f80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4424c1f50, 0xc44a255540, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4424c1f50, 0xc43adc00e0, 0x0, 0xc43e30b9b8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4424c1f50, 0xa74a80, 0xc43adc00e0, 0x7fad6035c8e0, 0xc4424c1f50, 0x1038500, 0xc4424c1f20)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4424c1f20, 0xa74a80, 0xc43adc00e0, 0xc44a255440, 0xc44a255460, 0x7fad6035c8e0, 0xc4424c1f20, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4424c1f20, 0xa74a80, 0xc43adc00e0, 0x7fad6035c8e0, 0xc4424c1f20, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4424c1f20, 0xa74a80, 0xc43adc00e0, 0xc44a255440, 0xc44a255480, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xffffffff, 0x0, 0x19, 0xc44a6737e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xc44a255440, 0xc44a2554a0, 0xc44a673878, 0x558012, 0x588a5810, 0xd29b3f4)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xed01c4f10, 0x0, 0x0, 0x739753)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xc44a255440, 0xc44a2554c0, 0xa91b40, 0xc44a2554e0, 0xc44a255440, 0xc44a2554e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xc420246100, 0xc441187f00, 0xc44a673a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xc44a255440, 0xc44a255460, 0x7abb, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43f9e3f80, 0xc4408703c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ece23fd)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0700, 0xc420331380, 0x10265c0, 0xc43f9e3fb0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0700, 0xc43f9e3fb0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0700)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10206 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4357a62d0, 0xc4357a6300, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4357a62d0, 0xc445cf3f00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4357a62d0, 0xc448b04c20, 0x0, 0xc4370bdd10, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4357a62d0, 0xa74a80, 0xc448b04c20, 0x7fad6035c8e0, 0xc4357a62d0, 0x1038500, 0xc4357a62a0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4357a62a0, 0xa74a80, 0xc448b04c20, 0xc445cf32c0, 0xc445cf32e0, 0x7fad6035c8e0, 0xc4357a62a0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4357a62a0, 0xa74a80, 0xc448b04c20, 0x7fad6035c8e0, 0xc4357a62a0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4357a62a0, 0xa74a80, 0xc448b04c20, 0xc445cf32c0, 0xc445cf3320, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xffffffff, 0x0, 0x19, 0xc44226f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xc445cf32c0, 0xc445cf3340, 0xc44226f878, 0x558012, 0x588a58c7, 0x3a488275)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xc44226f900, 0x0, 0x0, 0xf4)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xc445cf32c0, 0xc445cf3e80, 0xa91b40, 0xc445cf3ea0, 0xc445cf32c0, 0xc445cf3ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xc420246100, 0xc43714e850, 0xc44226fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xc445cf32c0, 0xc445cf32e0, 0xb442, 0xc420319b18, 0x0, 0xc420319b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43580eae0, 0xc431058ae0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ec6ce01)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc434d94a00, 0xc420331380, 0x10265c0, 0xc43580eb70, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc434d94a00, 0xc43580eb70)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc434d94a00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 977 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204895c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204895c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x4fbdccb03de0ad07)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a2e7920, 0xc422334c2d, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a2e6b10, 0xc42a2e7920, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a2e6b10, 0xc424223f80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a2e6b10, 0xc4244736e0, 0x0, 0xc42a2e1e68, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a2e6b10, 0xa74a80, 0xc4244736e0, 0x7fad6035c8e0, 0xc42a2e6b10, 0x1038500, 0xc42a2e6ae0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a2e6ae0, 0xa74a80, 0xc4244736e0, 0xc424233800, 0xc424233820, 0x7fad6035c8e0, 0xc42a2e6ae0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a2e6ae0, 0xa74a80, 0xc4244736e0, 0x7fad6035c8e0, 0xc42a2e6ae0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a2e6ae0, 0xa74a80, 0xc4244736e0, 0xc424233800, 0xc424233860, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xffffffff, 0x0, 0x19, 0xc4294757e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xc424233800, 0xc424233880, 0xc429475878, 0x558012, 0x588a5741, 0x3761df55)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xed01c4e41, 0x0, 0x0, 0x517a64)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xc424233800, 0xc4242338c0, 0xa91b40, 0xc4242338e0, 0xc424233800, 0xc4242338e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xc420246100, 0xc42a1b7f50, 0xc429475a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xc424233800, 0xc424233820, 0xc28a, 0xc42023bb18, 0x0, 0xc42023bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc425e39530, 0xc429d97920, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cadc00, 0xc420331380, 0x10265c0, 0xc42a2016b0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cadc00, 0xc42a2016b0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cadc00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8983 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43b9a7200, 0xc43b9a7230, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43b9a7200, 0xc4496c0920, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43b9a7200, 0xc43dd60000, 0x0, 0xc4425b0870, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43b9a7200, 0xa74a80, 0xc43dd60000, 0x7fad6035c8e0, 0xc43b9a7200, 0x1038500, 0xc43b9a71d0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43b9a71d0, 0xa74a80, 0xc43dd60000, 0xc4496c0820, 0xc4496c0840, 0x7fad6035c8e0, 0xc43b9a71d0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b9a71d0, 0xa74a80, 0xc43dd60000, 0x7fad6035c8e0, 0xc43b9a71d0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43b9a71d0, 0xa74a80, 0xc43dd60000, 0xc4496c0820, 0xc4496c0860, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xffffffff, 0x0, 0x19, 0xc44a6757e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xc4496c0820, 0xc4496c0880, 0xc44a675878, 0x558012, 0x588a5810, 0xd39d62f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xed01c4f10, 0x0, 0x0, 0x3b9748)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xc4496c0820, 0xc4496c08a0, 0xa91b40, 0xc4496c08c0, 0xc4496c0820, 0xc4496c08c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xc420246100, 0xc43e30ba10, 0xc44a675a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xc4496c0820, 0xc4496c0840, 0x8cd7, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43caf4390, 0xc4408709c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f400720)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0b00, 0xc420331380, 0x10265c0, 0xc43caf4420, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0b00, 0xc43caf4420)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0b00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 330 [select]:\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc420246600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855\n\ngoroutine 8920 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc449d8f350, 0xc449d8f380, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc449d8f350, 0xc44a4d00e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc449d8f350, 0xc44a505c40, 0x0, 0xc44479a7e0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc449d8f350, 0xa74a80, 0xc44a505c40, 0x7fad6035c8e0, 0xc449d8f350, 0x1038500, 0xc449d8f320)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc449d8f320, 0xa74a80, 0xc44a505c40, 0xc44a4e3fe0, 0xc44a4d0000, 0x7fad6035c8e0, 0xc449d8f320, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449d8f320, 0xa74a80, 0xc44a505c40, 0x7fad6035c8e0, 0xc449d8f320, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc449d8f320, 0xa74a80, 0xc44a505c40, 0xc44a4e3fe0, 0xc44a4d0020, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xffffffff, 0x0, 0x19, 0xc43d7c57e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xc44a4e3fe0, 0xc44a4d0040, 0xc43d7c5878, 0x558012, 0x588a57f5, 0x397ee00c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xed01c4ef5, 0x0, 0x0, 0x223e3)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xc44a4e3fe0, 0xc44a4d0060, 0xa91b40, 0xc44a4d0080, 0xc44a4e3fe0, 0xc44a4d0080)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xc420246100, 0xc445048110, 0xc43d7c5a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xc44a4e3fe0, 0xc44a4d0000, 0x16c8, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc449d8e9c0, 0xc42c213f80, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5986c5)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b200, 0xc420331380, 0x10265c0, 0xc449d8ea80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b200, 0xc449d8ea80)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9665 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43956cf30, 0xc43956cf60, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43956cf30, 0xc449c42860, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43956cf30, 0xc449e2a000, 0x0, 0xc43c1e6218, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43956cf30, 0xa74a80, 0xc449e2a000, 0x7fad6035c8e0, 0xc43956cf30, 0x1038500, 0xc43956cf00)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43956cf00, 0xa74a80, 0xc449e2a000, 0xc449c42760, 0xc449c42780, 0x7fad6035c8e0, 0xc43956cf00, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43956cf00, 0xa74a80, 0xc449e2a000, 0x7fad6035c8e0, 0xc43956cf00, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43956cf00, 0xa74a80, 0xc449e2a000, 0xc449c42760, 0xc449c427a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xffffffff, 0x0, 0x19, 0xc42eb397e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xc449c42760, 0xc449c427c0, 0xc42eb39878, 0x558012, 0x588a586a, 0x103e60fa)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xc42eb39900, 0x0, 0x0, 0x3f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xc449c42760, 0xc449c427e0, 0xa91b40, 0xc449c42800, 0xc449c42760, 0xc449c42800)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xc420246100, 0xc4446e8400, 0xc42eb39a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xc449c42760, 0xc449c42780, 0x75cf, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43dff13e0, 0xc441eb6000, 0xc420179d60, 0x0, 0x0, 0x7fad625dc000, 0xc41eac8542)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87b00, 0xc420331380, 0x10265c0, 0xc43dff1470, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87b00, 0xc43dff1470)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87b00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8735 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc443f03b60, 0xc443f03b90, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc443f03b60, 0xc4325bf180, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc443f03b60, 0xc43281e2e0, 0x0, 0xc443ff55d8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc443f03b60, 0xa74a80, 0xc43281e2e0, 0x7fad6035c8e0, 0xc443f03b60, 0x1038500, 0xc443f03b30)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc443f03b30, 0xa74a80, 0xc43281e2e0, 0xc4325bf000, 0xc4325bf020, 0x7fad6035c8e0, 0xc443f03b30, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc443f03b30, 0xa74a80, 0xc43281e2e0, 0x7fad6035c8e0, 0xc443f03b30, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc443f03b30, 0xa74a80, 0xc43281e2e0, 0xc4325bf000, 0xc4325bf040, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xffffffff, 0x0, 0x19, 0xc43e8bb7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xc4325bf000, 0xc4325bf060, 0xc43e8bb878, 0x558012, 0x588a57d7, 0x3a7e9931)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xed01c4ed7, 0x0, 0x0, 0x6463c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xc4325bf000, 0xc4325bf0a0, 0xa91b40, 0xc4325bf0e0, 0xc4325bf000, 0xc4325bf0e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xc420246100, 0xc4463ebf00, 0xc43e8bba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xc4325bf000, 0xc4325bf020, 0x4254, 0xc420e1ab18, 0x0, 0xc420e1ab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc443f02870, 0xc44661b980, 0xc420179d60, 0x0, 0x0, 0x10, 0xc42ac41698)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421001000, 0xc420331380, 0x10265c0, 0xc443f02990, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421001000, 0xc443f02990)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421001000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10207 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4356a7560, 0xc4356a75f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4356a7560, 0xc443e900c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4356a7560, 0xc447578260, 0x0, 0xc4370235a0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4356a7560, 0xa74a80, 0xc447578260, 0x7fad6035c8e0, 0xc4356a7560, 0x1038500, 0xc4356a74d0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4356a74d0, 0xa74a80, 0xc447578260, 0xc44a24ffa0, 0xc44a24ffc0, 0x7fad6035c8e0, 0xc4356a74d0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4356a74d0, 0xa74a80, 0xc447578260, 0x7fad6035c8e0, 0xc4356a74d0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4356a74d0, 0xa74a80, 0xc447578260, 0xc44a24ffa0, 0xc44a24ffe0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xffffffff, 0x0, 0x19, 0xc442c1d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xc44a24ffa0, 0xc443e90000, 0xc442c1d878, 0x558012, 0x588a58c7, 0x3a6a326f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xc442c1d900, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xc44a24ffa0, 0xc443e90020, 0xa91b40, 0xc443e90040, 0xc44a24ffa0, 0xc443e90040)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xc420246100, 0xc437237400, 0xc442c1da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xc44a24ffa0, 0xc44a24ffc0, 0xb624, 0xc420319b18, 0x0, 0xc420319b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43575dc20, 0xc4320cd260, 0xc420179d60, 0x0, 0x0, 0xc43b43a600, 0xb)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434d94e00, 0xc420331380, 0x10265c0, 0xc43575dc80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434d94e00, 0xc43575dc80)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434d94e00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8319 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4248f06c0, 0xc4248f06f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4248f06c0, 0xc437e1e300, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4248f06c0, 0xc437e3fbe0, 0x0, 0xc426fafd58, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4248f06c0, 0xa74a80, 0xc437e3fbe0, 0x7fad6035c8e0, 0xc4248f06c0, 0x1038500, 0xc4248f0690)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4248f0690, 0xa74a80, 0xc437e3fbe0, 0xc437e1e180, 0xc437e1e1c0, 0x7fad6035c8e0, 0xc4248f0690, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f0690, 0xa74a80, 0xc437e3fbe0, 0x7fad6035c8e0, 0xc4248f0690, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4248f0690, 0xa74a80, 0xc437e3fbe0, 0xc437e1e180, 0xc437e1e200, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xffffffff, 0x0, 0x19, 0xc4293e17e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xc437e1e180, 0xc437e1e220, 0xc4293e1878, 0x558012, 0x588a5798, 0xd9a631c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xed01c4e98, 0x0, 0x0, 0x4ca84e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xc437e1e180, 0xc437e1e260, 0xa91b40, 0xc437e1e280, 0xc437e1e180, 0xc437e1e280)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xc420246100, 0xc43d7955b0, 0xc4293e1a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xc437e1e180, 0xc437e1e1c0, 0x454f, 0xc420e1ab18, 0x0, 0xc420e1ab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42f407080, 0xc44363f560, 0xc420179d60, 0x0, 0x0, 0xc4202e17a8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67d00, 0xc420331380, 0x10265c0, 0xc42f4072c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67d00, 0xc42f4072c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67d00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9507 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4388fa1e0, 0xc4388fa210, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4388fa1e0, 0xc4472e70e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4388fa1e0, 0xc4476a40a0, 0x0, 0xc4393d2f50, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4388fa1e0, 0xa74a80, 0xc4476a40a0, 0x7fad6035c8e0, 0xc4388fa1e0, 0x1038500, 0xc4388fa1b0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4388fa1b0, 0xa74a80, 0xc4476a40a0, 0xc4472e6fe0, 0xc4472e7000, 0x7fad6035c8e0, 0xc4388fa1b0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4388fa1b0, 0xa74a80, 0xc4476a40a0, 0x7fad6035c8e0, 0xc4388fa1b0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4388fa1b0, 0xa74a80, 0xc4476a40a0, 0xc4472e6fe0, 0xc4472e7020, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xffffffff, 0x0, 0x19, 0xc449dad7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xc4472e6fe0, 0xc4472e7040, 0xc449dad878, 0x558012, 0x588a584f, 0x3b937843)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xed01c4f4f, 0x0, 0x0, 0x6225e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xc4472e6fe0, 0xc4472e7060, 0xa91b40, 0xc4472e7080, 0xc4472e6fe0, 0xc4472e7080)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xc420246100, 0xc43940c4e0, 0xc449dada70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xc4472e6fe0, 0xc4472e7000, 0x3352, 0xc42023ab18, 0x0, 0xc42023ab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43891e450, 0xc4377bcae0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ec3b394)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064d00, 0xc420331380, 0x10265c0, 0xc43891e480, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064d00, 0xc43891e480)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064d00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 5327 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048c504)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048c500)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x4dbfada6ed50c1e2)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc444637c20, 0xc420162418, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc444637320, 0xc444637c20, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc444637320, 0xc42b39d080, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc444637320, 0xc42b5a9ba0, 0x0, 0xc443fc3f78, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc444637320, 0xa74a80, 0xc42b5a9ba0, 0x7fad6035c8e0, 0xc444637320, 0x1038500, 0xc4446372f0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4446372f0, 0xa74a80, 0xc42b5a9ba0, 0xc42b39c780, 0xc42b39c7a0, 0x7fad6035c8e0, 0xc4446372f0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4446372f0, 0xa74a80, 0xc42b5a9ba0, 0x7fad6035c8e0, 0xc4446372f0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4446372f0, 0xa74a80, 0xc42b5a9ba0, 0xc42b39c780, 0xc42b39c7c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xffffffff, 0x0, 0x19, 0xc4359957e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xc42b39c780, 0xc42b39c7e0, 0xc435995878, 0x558012, 0x588a575f, 0x39a47310)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xc435995900, 0x0, 0x0, 0x3f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xc42b39c780, 0xc42b39c800, 0xa91b40, 0xc42b39c820, 0xc42b39c780, 0xc42b39c820)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xc420246100, 0xc444f0e800, 0xc435995a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xc42b39c780, 0xc42b39c7a0, 0x962f, 0xc420e1ab18, 0x0, 0xc420e1ab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc444f08f60, 0xc43bba4a20, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cb4000, 0xc420331380, 0x10265c0, 0xc444f08f90, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cb4000, 0xc444f08f90)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cb4000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9882 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437ceb530, 0xc437ceb590, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437ceb530, 0xc449d7dd40, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437ceb530, 0xc4410fc520, 0x0, 0xc438c38dd8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437ceb530, 0xa74a80, 0xc4410fc520, 0x7fad6035c8e0, 0xc437ceb530, 0x1038500, 0xc437ceb4d0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc437ceb4d0, 0xa74a80, 0xc4410fc520, 0xc449d7dc20, 0xc449d7dc40, 0x7fad6035c8e0, 0xc437ceb4d0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437ceb4d0, 0xa74a80, 0xc4410fc520, 0x7fad6035c8e0, 0xc437ceb4d0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc437ceb4d0, 0xa74a80, 0xc4410fc520, 0xc449d7dc20, 0xc449d7dc60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xffffffff, 0x0, 0x19, 0xc4414597e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xc449d7dc20, 0xc449d7dc80, 0xc441459878, 0x558012, 0x588a5888, 0x104d9a02)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xed01c4f88, 0x0, 0x0, 0x8fe8c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xc449d7dc20, 0xc449d7dcc0, 0xa91b40, 0xc449d7dce0, 0xc449d7dc20, 0xc449d7dce0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xc420246100, 0xc438d1e890, 0xc441459a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xc449d7dc20, 0xc449d7dc40, 0x600d, 0xc42023bb18, 0x0, 0xc42023bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc437eefd40, 0xc441196f00, 0xc420179d60, 0x0, 0x0, 0xc439252d40, 0x646e696b)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5300, 0xc420331380, 0x10265c0, 0xc437eefe60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5300, 0xc437eefe60)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9218 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc4469ffbc0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc43f716b10, 0xc4469ffbc0, 0x10343c0, 0xc4201e3040)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326\n\ngoroutine 12081 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4496097a0, 0xc4496097d0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4496097a0, 0xc4457267a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4496097a0, 0xc445b966a0, 0x0, 0xc427f8c868, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4496097a0, 0xa74a80, 0xc445b966a0, 0x7fad6035c8e0, 0xc4496097a0, 0x1038500, 0xc449609770)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc449609770, 0xa74a80, 0xc445b966a0, 0xc445726680, 0xc4457266a0, 0x7fad6035c8e0, 0xc449609770, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449609770, 0xa74a80, 0xc445b966a0, 0x7fad6035c8e0, 0xc449609770, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc449609770, 0xa74a80, 0xc445b966a0, 0xc445726680, 0xc4457266c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xffffffff, 0x0, 0x19, 0xc435e0b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xc445726680, 0xc4457266e0, 0xc435e0b878, 0x558012, 0x588a59f3, 0x37e5eaf5)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xed01c50f3, 0x0, 0x0, 0xdcd52)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xc445726680, 0xc445726700, 0xa91b40, 0xc445726720, 0xc445726680, 0xc445726720)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xc420246100, 0xc428dc6980, 0xc435e0ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xc445726680, 0xc4457266a0, 0x5845, 0xc435e50b18, 0x0, 0xc435e50b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442b52ff0, 0xc4445a82a0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f309dfd)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064b00, 0xc420331380, 0x10265c0, 0xc442b53020, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064b00, 0xc442b53020)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064b00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 329 [IO wait]:\nnet.runtime_pollWait(0x7fad6039d2a0, 0x72, 0x8)\n\t/usr/local/go/src/runtime/netpoll.go:160 +0x59\nnet.(*pollDesc).wait(0xc421d461b0, 0x72, 0xc4200458d8, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38\nnet.(*pollDesc).waitRead(0xc421d461b0, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34\nnet.(*netFD).Read(0xc421d46150, 0xc421fa0000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_unix.go:243 +0x1a1\nnet.(*conn).Read(0xc42019de50, 0xc421fa0000, 0x8000, 0x8000, 0x0, 0x0, 0x0)\n\t/usr/local/go/src/net/net.go:173 +0x70\nbufio.(*Reader).fill(0xc4210ec6c0)\n\t/usr/local/go/src/bufio/bufio.go:97 +0x10c\nbufio.(*Reader).Read(0xc4210ec6c0, 0xc420246578, 0x9, 0x9, 0x43fe80, 0xc43eaf22c0, 0xc4445b4e20)\n\t/usr/local/go/src/bufio/bufio.go:209 +0x1bc\nio.ReadAtLeast(0x102f380, 0xc4210ec6c0, 0xc420246578, 0x9, 0x9, 0x9, 0xc420045ae8, 0x451a90, 0xc438650840)\n\t/usr/local/go/src/io/io.go:307 +0xa4\nio.ReadFull(0x102f380, 0xc4210ec6c0, 0xc420246578, 0x9, 0x9, 0xc43eaf22c0, 0x1, 0x2)\n\t/usr/local/go/src/io/io.go:325 +0x58\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc420246578, 0x9, 0x9, 0x102f380, 0xc4210ec6c0, 0x0, 0xc400000000, 0xc420045c80, 0x7f3105)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420246540, 0x3895, 0x3895, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc421f520f0, 0xc4445f9a70, 0xc4445f9a70, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420246600, 0xc421f52180, 0xb475b8)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc420246600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc42019de50, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc42019de50)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac\n\ngoroutine 8922 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43ba15e90, 0xc43ba15ec0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43ba15e90, 0xc43ab20000, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43ba15e90, 0xc44a3ff520, 0x0, 0xc4424c5398, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43ba15e90, 0xa74a80, 0xc44a3ff520, 0x7fad6035c8e0, 0xc43ba15e90, 0x1038500, 0xc43ba15e60)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43ba15e60, 0xa74a80, 0xc44a3ff520, 0xc43ab49be0, 0xc43ab49c00, 0x7fad6035c8e0, 0xc43ba15e60, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43ba15e60, 0xa74a80, 0xc44a3ff520, 0x7fad6035c8e0, 0xc43ba15e60, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43ba15e60, 0xa74a80, 0xc44a3ff520, 0xc43ab49be0, 0xc43ab49c20, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xffffffff, 0x0, 0x19, 0xc43ed817e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xc43ab49be0, 0xc43ab49c40, 0xc43ed81878, 0x558012, 0x588a57f5, 0x3a6fc265)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xed01c4ef5, 0x0, 0x0, 0x62db0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xc43ab49be0, 0xc43ab49c60, 0xa91b40, 0xc43ab49c80, 0xc43ab49be0, 0xc43ab49c80)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xc420246100, 0xc442f6c840, 0xc43ed81a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xc43ab49be0, 0xc43ab49c00, 0x49bc, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4445c4f00, 0xc42b6f2540, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5986c5)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b800, 0xc420331380, 0x10265c0, 0xc4445c4f30, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b800, 0xc4445c4f30)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b800)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 1182 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204962c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204962c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42cdd1d20, 0x3, 0x3, 0xc42a6df448, 0x0, 0xc42ccadac0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42cc96e70, 0x159dc604330, 0x159dc64d710, 0xc42cdd1d20, 0x3, 0x3, 0xc42cc96e00, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42cc96e70, 0xc42cc96d50, 0x0, 0xc42cdf6038, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42cc96e70, 0xa934c0, 0xc42cc96d50, 0x7fad6035c8e0, 0xc42cc96e70, 0x1038500, 0xc42cc96e40)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42cc96e40, 0xa934c0, 0xc42cc96d50, 0xc42cdd1c20, 0xc42cdd1c40, 0x7fad6035c8e0, 0xc42cc96e40, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cc96e40, 0xa934c0, 0xc42cc96d50, 0x7fad6035c8e0, 0xc42cc96e40, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42cc96e40, 0xa934c0, 0xc42cc96d50, 0xc42cdd1c20, 0xc42cdd1c60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xffffffff, 0x0, 0x19, 0xc42a6df7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xc42cdd1c20, 0xc42cdd1c80, 0xc42a6df878, 0x558012, 0x588a5743, 0x12b73ef2)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xed01c4e43, 0x0, 0x0, 0x9e8d)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xc42cdd1c20, 0xc42cdd1ca0, 0xa91b40, 0xc42cdd1cc0, 0xc42cdd1c20, 0xc42cdd1cc0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xc420246100, 0xc42cc87fe0, 0xc42a6dfa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xc42cdd1c20, 0xc42cdd1c40, 0x65, 0xc425813b18, 0x0, 0xc425813b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42cc96c30, 0xc42cb7a4e0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41fa28d46)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429a83900, 0xc420331380, 0x10265d8, 0xc42cc96cf0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429a83900, 0xc42cc96cf0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429a83900)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 871 [select, 13 minutes]:\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc426048300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855\n\ngoroutine 11636 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc447654990, 0xc4476549c0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc447654990, 0xc44289f180, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc447654990, 0xc44289e6c0, 0x0, 0xc42f98c890, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc447654990, 0xa74a80, 0xc44289e6c0, 0x7fad6035c8e0, 0xc447654990, 0x1038500, 0xc447654960)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc447654960, 0xa74a80, 0xc44289e6c0, 0xc44289f080, 0xc44289f0a0, 0x7fad6035c8e0, 0xc447654960, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447654960, 0xa74a80, 0xc44289e6c0, 0x7fad6035c8e0, 0xc447654960, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc447654960, 0xa74a80, 0xc44289e6c0, 0xc44289f080, 0xc44289f0c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xffffffff, 0x0, 0x19, 0xc439fb77e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xc44289f080, 0xc44289f0e0, 0xc439fb7878, 0x558012, 0x588a5999, 0x3aa461f1)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xed01c5099, 0x0, 0x0, 0x14ecf)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xc44289f080, 0xc44289f100, 0xa91b40, 0xc44289f120, 0xc44289f080, 0xc44289f120)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xc420246100, 0xc42f9bfc80, 0xc439fb7a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xc44289f080, 0xc44289f0a0, 0x859, 0xc440d62b18, 0x0, 0xc440d62b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc447654420, 0xc443093800, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f20a56f)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2800, 0xc420331380, 0x10265c0, 0xc4476544e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2800, 0xc4476544e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2800)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8921 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4445c4030, 0xc4445c4060, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4445c4030, 0xc44a445f20, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4445c4030, 0xc44a4d05c0, 0x0, 0xc442f6c770, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4445c4030, 0xa74a80, 0xc44a4d05c0, 0x7fad6035c8e0, 0xc4445c4030, 0x1038500, 0xc4445c4000)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4445c4000, 0xa74a80, 0xc44a4d05c0, 0xc44a445e20, 0xc44a445e40, 0x7fad6035c8e0, 0xc4445c4000, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4445c4000, 0xa74a80, 0xc44a4d05c0, 0x7fad6035c8e0, 0xc4445c4000, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4445c4000, 0xa74a80, 0xc44a4d05c0, 0xc44a445e20, 0xc44a445e60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xffffffff, 0x0, 0x19, 0xc43ed7f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xc44a445e20, 0xc44a445e80, 0xc43ed7f878, 0x558012, 0x588a57f5, 0x39fd530a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xed01c4ef5, 0x0, 0x0, 0x47d77)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xc44a445e20, 0xc44a445ea0, 0xa91b40, 0xc44a445ec0, 0xc44a445e20, 0xc44a445ec0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xc420246100, 0xc44479a890, 0xc43ed7fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xc44a445e20, 0xc44a445e40, 0x31bd, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43a29d440, 0xc42b6f23c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5986c5)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b500, 0xc420331380, 0x10265c0, 0xc43a29d470, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b500, 0xc43a29d470)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b500)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10275 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43562b590, 0xc43562b5f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43562b590, 0xc43ee00c60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43562b590, 0xc43b3f6e40, 0x0, 0xc436f58488, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43562b590, 0xa74a80, 0xc43b3f6e40, 0x7fad6035c8e0, 0xc43562b590, 0x1038500, 0xc43562b560)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43562b560, 0xa74a80, 0xc43b3f6e40, 0xc43ee00b40, 0xc43ee00b60, 0x7fad6035c8e0, 0xc43562b560, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43562b560, 0xa74a80, 0xc43b3f6e40, 0x7fad6035c8e0, 0xc43562b560, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43562b560, 0xa74a80, 0xc43b3f6e40, 0xc43ee00b40, 0xc43ee00b80, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xffffffff, 0x0, 0x19, 0xc442c197e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xc43ee00b40, 0xc43ee00ba0, 0xc442c19878, 0x558012, 0x588a58c7, 0x3b8f70bd)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xed01c4fc7, 0x0, 0x0, 0x12a6b9)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xc43ee00b40, 0xc43ee00bc0, 0xa91b40, 0xc43ee00be0, 0xc43ee00b40, 0xc43ee00be0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xc420246100, 0xc4370bdf00, 0xc442c19a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xc43ee00b40, 0xc43ee00b60, 0x97be, 0xc420319b18, 0x0, 0xc420319b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43564ef00, 0xc42fa64240, 0xc420179d60, 0x0, 0x0, 0x10, 0xc43e5a21d8)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdca00, 0xc420331380, 0x10265c0, 0xc43564ef60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdca00, 0xc43564ef60)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdca00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9249 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc42a8fcf00, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f\ngithub.com/weaveworks/cortex/ingester.(*Ingester).LabelValues(0xc4201e3040, 0x7fad6035c8e0, 0xc42a8fcf00, 0xc43f717bb0, 0x0, 0xc43f717c68, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:298 +0x60\ngithub.com/weaveworks/cortex._Ingester_LabelValues_Handler.func1(0x7fad6035c8e0, 0xc42a8fcf00, 0xa73d60, 0xc43f717bb0, 0x7fad6035c8e0, 0xc42a8fcf00, 0x1038500, 0xc42a8fced0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:457 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a8fced0, 0xa73d60, 0xc43f717bb0, 0xc43698e580, 0xc43698e5a0, 0x7fad6035c8e0, 0xc42a8fced0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fced0, 0xa73d60, 0xc43f717bb0, 0x7fad6035c8e0, 0xc42a8fced0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a8fced0, 0xa73d60, 0xc43f717bb0, 0xc43698e580, 0xc43698e5c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0xffffffff, 0x0, 0x19, 0xc4441d97e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0xc43698e580, 0xc43698e5e0, 0xc4441d9878, 0x558012, 0x588a5827, 0xf0d8b43)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0x41167e, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0xc43698e580, 0xc43698e600, 0xa91b40, 0xc43698e640, 0xc43698e580, 0xc43698e640)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0xc420246100, 0xc43f717b70, 0xc4441d9a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0xc43698e580, 0xc43698e5a0, 0xa, 0xc42023bb18, 0x0, 0xc42023bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_LabelValues_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a8fcdb0, 0xc4415b5320, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:459 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc4390fb200, 0xc420331380, 0x10265f0, 0xc42a8fce70, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc4390fb200, 0xc42a8fce70)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc440f5ebb0, 0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc4390fb200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 12196 [chan receive, 2 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc4485843c0, 0xc43cb882d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47\nnet/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc4485843c0, 0xc43cb882d0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc4485843c0, 0xc43cb882d0)\n\t/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc441f95760, 0xc43cb882d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab\nnet/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc441f95760, 0xc43cb882d0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc441f95760, 0xc43cb882d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc441f95740, 0xc43cb880f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2\nnet/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc441f95740, 0xc43cb880f0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc441f95720, 0xc43cb880f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c\nnet/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc441f95720, 0xc43cb880f0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc42bd16340, 0xc43cb880f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449\nnet/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc42bd16340, 0xc43cb88000)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc42bd16340, 0xc43cb88000)\n\t/usr/local/go/src/net/http/server.go:2202 +0x7d\nnet/http.(*conn).serve(0xc42854fb00, 0x1038440, 0xc438720880)\n\t/usr/local/go/src/net/http/server.go:1579 +0x4b7\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:2293 +0x44d\n\ngoroutine 8473 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc429a59aa0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc42bd13430, 0xc429a59aa0, 0x10343c0, 0xc4201e3040)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326\n\ngoroutine 1271 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204895c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204895c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x67d0eb2b1a80b1fc)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42d39dc80, 0xc42939e0cd, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d39dc50, 0xc42d39dc80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d39dc50, 0xc42d4c9240, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d39dc50, 0xc42d4c9080, 0x0, 0xc42d2c7f90, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d39dc50, 0xa74a80, 0xc42d4c9080, 0x7fad6035c8e0, 0xc42d39dc50, 0x1038500, 0xc42d39dc20)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d39dc20, 0xa74a80, 0xc42d4c9080, 0xc42d4c9140, 0xc42d4c9160, 0x7fad6035c8e0, 0xc42d39dc20, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d39dc20, 0xa74a80, 0xc42d4c9080, 0x7fad6035c8e0, 0xc42d39dc20, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d39dc20, 0xa74a80, 0xc42d4c9080, 0xc42d4c9140, 0xc42d4c9180, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xffffffff, 0x0, 0x19, 0xc4215db7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xc42d4c9140, 0xc42d4c91a0, 0xc4215db878, 0x558012, 0x588a5743, 0x2059cadd)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xed01c4e43, 0x0, 0x0, 0x7bb2)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xc42d4c9140, 0xc42d4c91c0, 0xa91b40, 0xc42d4c91e0, 0xc42d4c9140, 0xc42d4c91e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xc420246100, 0xc42d2c7f20, 0xc4215dba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xc42d4c9140, 0xc42d4c9160, 0x73, 0xc42023eb18, 0x0, 0xc42023eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d39daa0, 0xc42d524000, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d3b0300, 0xc420331380, 0x10265c0, 0xc42d39db60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d3b0300, 0xc42d39db60)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d3b0300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8327 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4292daa50, 0xc4292daa80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4292daa50, 0xc437b61380, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4292daa50, 0xc4383a2080, 0x0, 0xc4292d99f0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4292daa50, 0xa74a80, 0xc4383a2080, 0x7fad6035c8e0, 0xc4292daa50, 0x1038500, 0xc4292daa20)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4292daa20, 0xa74a80, 0xc4383a2080, 0xc437b61260, 0xc437b61280, 0x7fad6035c8e0, 0xc4292daa20, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4292daa20, 0xa74a80, 0xc4383a2080, 0x7fad6035c8e0, 0xc4292daa20, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4292daa20, 0xa74a80, 0xc4383a2080, 0xc437b61260, 0xc437b612a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xffffffff, 0x0, 0x19, 0xc439ee17e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xc437b61260, 0xc437b612c0, 0xc439ee1878, 0x558012, 0x588a579b, 0x32eefbdf)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xed01c4e9b, 0x0, 0x0, 0x11936e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xc437b61260, 0xc437b612e0, 0xa91b40, 0xc437b61300, 0xc437b61260, 0xc437b61300)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xc420246100, 0xc440d25d10, 0xc439ee1a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xc437b61260, 0xc437b61280, 0x9dd5, 0xc420e1db18, 0x0, 0xc420e1db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4223bde30, 0xc44088e0c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f40378e)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc426516400, 0xc420331380, 0x10265c0, 0xc4223bde60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc426516400, 0xc4223bde60)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc426516400)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 1019 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204962c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204962c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc423bd8e80, 0x4, 0x4, 0xc4293e3448, 0x0, 0xc42a49fac0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42a49df80, 0x159dc604330, 0x159dc64d710, 0xc423bd8e80, 0x4, 0x4, 0xc42a49df00, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42a49df80, 0xc42a49de30, 0x0, 0xc42a4d2700, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42a49df80, 0xa934c0, 0xc42a49de30, 0x7fad6035c8e0, 0xc42a49df80, 0x1038500, 0xc42a49df50)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a49df50, 0xa934c0, 0xc42a49de30, 0xc423bd8c00, 0xc423bd8c60, 0x7fad6035c8e0, 0xc42a49df50, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a49df50, 0xa934c0, 0xc42a49de30, 0x7fad6035c8e0, 0xc42a49df50, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a49df50, 0xa934c0, 0xc42a49de30, 0xc423bd8c00, 0xc423bd8ca0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xffffffff, 0x0, 0x19, 0xc4293e37e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xc423bd8c00, 0xc423bd8ce0, 0xc4293e3878, 0x558012, 0x588a5742, 0x166f6bd)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xed01c4e42, 0x0, 0x0, 0xdea7)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xc423bd8c00, 0xc423bd8d20, 0xa91b40, 0xc423bd8d60, 0xc423bd8c00, 0xc423bd8d60)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xc420246100, 0xc42a4d2690, 0xc4293e3a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xc423bd8c00, 0xc423bd8c60, 0x83, 0xc420317b18, 0x0, 0xc420317b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a49dd10, 0xc429975f20, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc4204ffe00, 0xc420331380, 0x10265d8, 0xc42a49ddd0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc4204ffe00, 0xc42a49ddd0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc4204ffe00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 941 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048e904)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048e900)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x9c30b4199368dad5)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a0776b0, 0xc420162418, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a076540, 0xc42a0776b0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a076540, 0xc425157e20, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a076540, 0xc425d81580, 0x0, 0xc42a074e78, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a076540, 0xa74a80, 0xc425d81580, 0x7fad6035c8e0, 0xc42a076540, 0x1038500, 0xc42a076510)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a076510, 0xa74a80, 0xc425d81580, 0xc4251562a0, 0xc4251562e0, 0x7fad6035c8e0, 0xc42a076510, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a076510, 0xa74a80, 0xc425d81580, 0x7fad6035c8e0, 0xc42a076510, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a076510, 0xa74a80, 0xc425d81580, 0xc4251562a0, 0xc425156320, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xffffffff, 0x0, 0x19, 0xc4215b97e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xc4251562a0, 0xc425156360, 0xc4215b9878, 0x558012, 0x588a5741, 0x35add981)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xed01c4e41, 0x0, 0x0, 0x27b95e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xc4251562a0, 0xc4251563a0, 0xa91b40, 0xc4251563e0, 0xc4251562a0, 0xc4251563e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xc420246100, 0xc4266315b0, 0xc4215b9a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xc4251562a0, 0xc4251562e0, 0x85f8, 0xc42023bb18, 0x0, 0xc42023bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc429fabd10, 0xc4294e3260, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0800, 0xc420331380, 0x10265c0, 0xc429fabd40, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0800, 0xc429fabd40)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0800)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8737 [chan send, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc44776e870, 0xc443c34180, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 12195 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43cf98450, 0xc43cf98480, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43cf98450, 0xc4400f5f40, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43cf98450, 0xc4457276a0, 0x0, 0xc4231ee1f8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43cf98450, 0xa74a80, 0xc4457276a0, 0x7fad6035c8e0, 0xc43cf98450, 0x1038500, 0xc43cf98420)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43cf98420, 0xa74a80, 0xc4457276a0, 0xc4400f5e20, 0xc4400f5e40, 0x7fad6035c8e0, 0xc43cf98420, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43cf98420, 0xa74a80, 0xc4457276a0, 0x7fad6035c8e0, 0xc43cf98420, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43cf98420, 0xa74a80, 0xc4457276a0, 0xc4400f5e20, 0xc4400f5e60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xffffffff, 0x0, 0x19, 0xc4359277e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xc4400f5e20, 0xc4400f5ea0, 0xc435927878, 0x558012, 0x588a59f4, 0x99738f8)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xed01c50f4, 0x0, 0x0, 0x140cd0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xc4400f5e20, 0xc4400f5ec0, 0xa91b40, 0xc4400f5ee0, 0xc4400f5e20, 0xc4400f5ee0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xc420246100, 0xc427f8d760, 0xc435927a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xc4400f5e20, 0xc4400f5e40, 0x7532, 0xc435e51b18, 0x0, 0xc435e51b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc449610d80, 0xc4445a8480, 0xc420179d60, 0x0, 0x0, 0x2b9de3948fa81040, 0x720a0a5b0a02cd0a)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065500, 0xc420331380, 0x10265c0, 0xc449610db0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065500, 0xc449610db0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065500)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 993 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048e0c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048e0c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x8ca9f4c8f66dcb26)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a1a19e0, 0xc420162418, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a1a17d0, 0xc42a1a19e0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a1a17d0, 0xc4244c0940, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a1a17d0, 0xc425120e20, 0x0, 0xc42a1b7488, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a1a17d0, 0xa74a80, 0xc425120e20, 0x7fad6035c8e0, 0xc42a1a17d0, 0x1038500, 0xc42a1a17a0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a1a17a0, 0xa74a80, 0xc425120e20, 0xc4244c05e0, 0xc4244c0600, 0x7fad6035c8e0, 0xc42a1a17a0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a1a17a0, 0xa74a80, 0xc425120e20, 0x7fad6035c8e0, 0xc42a1a17a0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a1a17a0, 0xa74a80, 0xc425120e20, 0xc4244c05e0, 0xc4244c0620, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xffffffff, 0x0, 0x19, 0xc42910f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xc4244c05e0, 0xc4244c0640, 0xc42910f878, 0x558012, 0x588a5741, 0x36482006)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xed01c4e41, 0x0, 0x0, 0x825ce)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xc4244c05e0, 0xc4244c0680, 0xa91b40, 0xc4244c06a0, 0xc4244c05e0, 0xc4244c06a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xc420246100, 0xc429fad030, 0xc42910fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xc4244c05e0, 0xc4244c0600, 0x6efd, 0xc427ba0b18, 0x0, 0xc427ba0b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a196120, 0xc429d97080, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4207b3300, 0xc420331380, 0x10265c0, 0xc42a196150, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4207b3300, 0xc42a196150)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4207b3300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 1190 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204962c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204962c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42c9402f0, 0x2, 0x2, 0xc42a6e3448, 0x0, 0xc42c9371c0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42c9452f0, 0x159dc604330, 0x159dc64d710, 0xc42c9402f0, 0x2, 0x2, 0xc42c945200, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42c9452f0, 0xc42c945200, 0x0, 0xc42c9402c8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42c9452f0, 0xa934c0, 0xc42c945200, 0x7fad6035c8e0, 0xc42c9452f0, 0x1038500, 0xc42c9452c0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c9452c0, 0xa934c0, 0xc42c945200, 0xc42c946f80, 0xc42c946fa0, 0x7fad6035c8e0, 0xc42c9452c0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9452c0, 0xa934c0, 0xc42c945200, 0x7fad6035c8e0, 0xc42c9452c0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c9452c0, 0xa934c0, 0xc42c945200, 0xc42c946f80, 0xc42c946fc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xffffffff, 0x0, 0x19, 0xc42a6e37e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xc42c946f80, 0xc42c946fe0, 0xc42a6e3878, 0x558012, 0x588a5743, 0x8c40078)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xc42a6e3900, 0x0, 0x0, 0x7f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xc42c946f80, 0xc42c947000, 0xa91b40, 0xc42c947020, 0xc42c946f80, 0xc42c947020)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xc420246100, 0xc42c940280, 0xc42a6e3a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xc42c946f80, 0xc42c946fa0, 0x4e, 0xc420238b18, 0x0, 0xc420238b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42c9450e0, 0xc42ca5c060, 0xc420179d60, 0x0, 0x0, 0xc42ad6bfa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc428c67b00, 0xc420331380, 0x10265d8, 0xc42c9451a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc428c67b00, 0xc42c9451a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc428c67b00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8990 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc442348510, 0xc442348540, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc442348510, 0xc435ba7860, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc442348510, 0xc44693fdc0, 0x0, 0xc43c1e6de8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc442348510, 0xa74a80, 0xc44693fdc0, 0x7fad6035c8e0, 0xc442348510, 0x1038500, 0xc4423484e0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4423484e0, 0xa74a80, 0xc44693fdc0, 0xc435ba7740, 0xc435ba7760, 0x7fad6035c8e0, 0xc4423484e0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4423484e0, 0xa74a80, 0xc44693fdc0, 0x7fad6035c8e0, 0xc4423484e0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4423484e0, 0xa74a80, 0xc44693fdc0, 0xc435ba7740, 0xc435ba77a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xffffffff, 0x0, 0x19, 0xc44a4917e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xc435ba7740, 0xc435ba77c0, 0xc44a491878, 0x558012, 0x588a5814, 0x868da4c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xc44a491900, 0x0, 0x0, 0xbf)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xc435ba7740, 0xc435ba77e0, 0xa91b40, 0xc435ba7800, 0xc435ba7740, 0xc435ba7800)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xc420246100, 0xc448040e30, 0xc44a491a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xc435ba7740, 0xc435ba7760, 0x33e1, 0xc420e1db18, 0x0, 0xc420e1db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc441afb830, 0xc443c35f20, 0xc420179d60, 0x0, 0x0, 0xc420252800, 0x450280)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1d00, 0xc420331380, 0x10265c0, 0xc441afb8f0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1d00, 0xc441afb8f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1d00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8916 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43c59b5c0, 0xc43c59b5f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43c59b5c0, 0xc44197c5a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43c59b5c0, 0xc43ecd1c00, 0x0, 0xc42df97b90, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43c59b5c0, 0xa74a80, 0xc43ecd1c00, 0x7fad6035c8e0, 0xc43c59b5c0, 0x1038500, 0xc43c59b590)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43c59b590, 0xa74a80, 0xc43ecd1c00, 0xc44197c4a0, 0xc44197c4c0, 0x7fad6035c8e0, 0xc43c59b590, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c59b590, 0xa74a80, 0xc43ecd1c00, 0x7fad6035c8e0, 0xc43c59b590, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43c59b590, 0xa74a80, 0xc43ecd1c00, 0xc44197c4a0, 0xc44197c4e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xffffffff, 0x0, 0x19, 0xc44abab7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xc44197c4a0, 0xc44197c500, 0xc44abab878, 0x558012, 0x588a57f2, 0xed7e733)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xed01c4ef2, 0x0, 0x0, 0x51cec9)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xc44197c4a0, 0xc44197c520, 0xa91b40, 0xc44197c540, 0xc44197c4a0, 0xc44197c540)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xc420246100, 0xc44609cc80, 0xc44ababa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xc44197c4a0, 0xc44197c4c0, 0x81a7, 0xc420049b18, 0x0, 0xc420049b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442d73b00, 0xc442ba8ea0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f0978d1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5a700, 0xc420331380, 0x10265c0, 0xc442d73b30, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5a700, 0xc442d73b30)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5a700)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8681 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc436c44600)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc4386531d0, 0xc436c44600, 0x10343c0, 0xc4201e3040)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326\n\ngoroutine 8225 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42ce529f0, 0xc42ce52a20, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42ce529f0, 0xc4379f4b40, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42ce529f0, 0xc437a66c60, 0x0, 0xc42ce48808, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42ce529f0, 0xa74a80, 0xc437a66c60, 0x7fad6035c8e0, 0xc42ce529f0, 0x1038500, 0xc42ce529c0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42ce529c0, 0xa74a80, 0xc437a66c60, 0xc4379f4a00, 0xc4379f4a20, 0x7fad6035c8e0, 0xc42ce529c0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42ce529c0, 0xa74a80, 0xc437a66c60, 0x7fad6035c8e0, 0xc42ce529c0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42ce529c0, 0xa74a80, 0xc437a66c60, 0xc4379f4a00, 0xc4379f4a40, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0xffffffff, 0x0, 0x19, 0xc43a66b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0xc4379f4a00, 0xc4379f4a80, 0xc43a66b878, 0x558012, 0x588a579b, 0x36f30836)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0x411715, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0xc4379f4a00, 0xc4379f4aa0, 0xa91b40, 0xc4379f4ac0, 0xc4379f4a00, 0xc4379f4ac0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0xc420246100, 0xc42d311090, 0xc43a66ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0xc4379f4a00, 0xc4379f4a20, 0x8844, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc429c871d0, 0xc42ba4d260, 0xc420179d60, 0x0, 0x0, 0xc43ceb47a8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42cbace00, 0xc420331380, 0x10265c0, 0xc429c87200, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42cbace00, 0xc429c87200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42cbace00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 961 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc420497484)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc420497480)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x30c7ec3b1c6a8f83)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a4fcd80, 0xc420162418, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a4fc600, 0xc42a4fcd80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a4fc600, 0xc423c7b2c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a4fc600, 0xc423d58e80, 0x0, 0xc42a4fb680, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a4fc600, 0xa74a80, 0xc423d58e80, 0x7fad6035c8e0, 0xc42a4fc600, 0x1038500, 0xc42a4fc5d0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a4fc5d0, 0xa74a80, 0xc423d58e80, 0xc423c7a000, 0xc423c7a040, 0x7fad6035c8e0, 0xc42a4fc5d0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4fc5d0, 0xa74a80, 0xc423d58e80, 0x7fad6035c8e0, 0xc42a4fc5d0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a4fc5d0, 0xa74a80, 0xc423d58e80, 0xc423c7a000, 0xc423c7a080, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xffffffff, 0x0, 0x19, 0xc4260f57e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xc423c7a000, 0xc423c7a0e0, 0xc4260f5878, 0x558012, 0x588a5741, 0x3a26f8f6)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xed01c4e41, 0x0, 0x0, 0xf6ff9)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xc423c7a000, 0xc423c7a180, 0xa91b40, 0xc423c7a1c0, 0xc423c7a000, 0xc423c7a1c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xc420246100, 0xc42a075ba0, 0xc4260f5a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xc423c7a000, 0xc423c7a040, 0xa7b1, 0xc420319b18, 0x0, 0xc420319b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a4d8150, 0xc42a1de300, 0xc420179d60, 0x0, 0x0, 0x312d70691c126564, 0x2d312d30322d3237)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1600, 0xc420331380, 0x10265c0, 0xc42a4d8180, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1600, 0xc42a4d8180)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 929 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048df04)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048df00)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0xe11e7b125b1afd74)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d1342f0, 0x2, 0x2, 0xc42d1f7448, 0x0, 0xc42d1363c0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1e2750, 0x159dc604330, 0x159dc64d710, 0xc42d1342f0, 0x2, 0x2, 0xc42d1e2700, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1e2750, 0xc42d1e2690, 0x0, 0xc42d1342c8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d1e2750, 0xa934c0, 0xc42d1e2690, 0x7fad6035c8e0, 0xc42d1e2750, 0x1038500, 0xc42d1e2720)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d1e2720, 0xa934c0, 0xc42d1e2690, 0xc42d12a600, 0xc42d12a620, 0x7fad6035c8e0, 0xc42d1e2720, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1e2720, 0xa934c0, 0xc42d1e2690, 0x7fad6035c8e0, 0xc42d1e2720, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d1e2720, 0xa934c0, 0xc42d1e2690, 0xc42d12a600, 0xc42d12a640, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xffffffff, 0x0, 0x19, 0xc42d1f77e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xc42d12a600, 0xc42d12a660, 0xc42d1f7878, 0x558012, 0x588a5743, 0x1a23da29)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xed01c4e43, 0x0, 0x0, 0x528a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xc42d12a600, 0xc42d12a680, 0xa91b40, 0xc42d12a6a0, 0xc42d12a600, 0xc42d12a6a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xc420246100, 0xc42d134280, 0xc42d1f7a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xc42d12a600, 0xc42d12a620, 0x43, 0xc420314b18, 0x0, 0xc420314b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d1e2570, 0xc42d1ea360, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ec100, 0xc420331380, 0x10265d8, 0xc42d1e2630, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ec100, 0xc42d1e2630)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ec100)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 958 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048e0c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048e0c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x5080f3cc323e99c)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a2011d0, 0xc422334c2d, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a200570, 0xc42a2011d0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a200570, 0xc4243f37c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a200570, 0xc4244b0920, 0x0, 0xc42a204508, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a200570, 0xa74a80, 0xc4244b0920, 0x7fad6035c8e0, 0xc42a200570, 0x1038500, 0xc42a200540)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a200540, 0xa74a80, 0xc4244b0920, 0xc4243f2a20, 0xc4243f2a40, 0x7fad6035c8e0, 0xc42a200540, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a200540, 0xa74a80, 0xc4244b0920, 0x7fad6035c8e0, 0xc42a200540, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a200540, 0xa74a80, 0xc4244b0920, 0xc4243f2a20, 0xc4243f2a60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xffffffff, 0x0, 0x19, 0xc4200577e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xc4243f2a20, 0xc4243f2a80, 0xc420057878, 0x558012, 0x588a5741, 0x36f3bd26)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xed01c4e41, 0x0, 0x0, 0x1d184f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xc4243f2a20, 0xc4243f2ac0, 0xa91b40, 0xc4243f2ae0, 0xc4243f2a20, 0xc4243f2ae0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xc420246100, 0xc42a15cc20, 0xc420057a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xc4243f2a20, 0xc4243f2a40, 0xaa72, 0xc42023bb18, 0x0, 0xc42023bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a0779e0, 0xc4288218c0, 0xc420179d60, 0x0, 0x0, 0x312d70691c126564, 0x2d312d30322d3237)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421ed1300, 0xc420331380, 0x10265c0, 0xc42a077a10, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421ed1300, 0xc42a077a10)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421ed1300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 999 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048df04)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048df00)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x3bd0391af88994ef)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc429e0f8c0, 0xc422334c2d, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc429e0ec60, 0xc429e0f8c0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc429e0ec60, 0xc42559ce80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc429e0ec60, 0xc4256362a0, 0x0, 0xc429e11170, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc429e0ec60, 0xa74a80, 0xc4256362a0, 0x7fad6035c8e0, 0xc429e0ec60, 0x1038500, 0xc429e0ec30)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc429e0ec30, 0xa74a80, 0xc4256362a0, 0xc42559c2c0, 0xc42559c340, 0x7fad6035c8e0, 0xc429e0ec30, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429e0ec30, 0xa74a80, 0xc4256362a0, 0x7fad6035c8e0, 0xc429e0ec30, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc429e0ec30, 0xa74a80, 0xc4256362a0, 0xc42559c2c0, 0xc42559c380, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xffffffff, 0x0, 0x19, 0xc4215b77e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xc42559c2c0, 0xc42559c3a0, 0xc4215b7878, 0x558012, 0x588a5741, 0x333ed171)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xed01c4e41, 0x0, 0x0, 0x12f6df)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xc42559c2c0, 0xc42559c3c0, 0xa91b40, 0xc42559c3e0, 0xc42559c2c0, 0xc42559c3e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xc420246100, 0xc429d4f450, 0xc4215b7a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xc42559c2c0, 0xc42559c340, 0xb217, 0xc42023db18, 0x0, 0xc42023db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc429dd1bc0, 0xc429d96060, 0xc420179d60, 0x0, 0x0, 0x1265646f6e040a24, 0x2d3237312d70691c)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb5600, 0xc420331380, 0x10265c0, 0xc429dd1bf0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb5600, 0xc429dd1bf0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb5600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 12390 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43bef8a80, 0xc43bef8ab0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43bef8a80, 0xc445db5160, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43bef8a80, 0xc43a890080, 0x0, 0xc43ce48d30, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43bef8a80, 0xa74a80, 0xc43a890080, 0x7fad6035c8e0, 0xc43bef8a80, 0x1038500, 0xc43bef8a50)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43bef8a50, 0xa74a80, 0xc43a890080, 0xc445db5060, 0xc445db5080, 0x7fad6035c8e0, 0xc43bef8a50, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43bef8a50, 0xa74a80, 0xc43a890080, 0x7fad6035c8e0, 0xc43bef8a50, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43bef8a50, 0xa74a80, 0xc43a890080, 0xc445db5060, 0xc445db50a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xffffffff, 0x0, 0x19, 0xc4377e37e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xc445db5060, 0xc445db50c0, 0xc4377e3878, 0x558012, 0x588a5a0e, 0x11403c0f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xed01c510e, 0x0, 0x0, 0x1ed2d7)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xc445db5060, 0xc445db50e0, 0xa91b40, 0xc445db5100, 0xc445db5060, 0xc445db5100)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xc420246100, 0xc44930f9c0, 0xc4377e3a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xc445db5060, 0xc445db5080, 0x7b07, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43dce3920, 0xc4377bdb00, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1600, 0xc420331380, 0x10265c0, 0xc43dce3950, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1600, 0xc43dce3950)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 11634 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441572de0, 0xc441572e10, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441572de0, 0xc4445abb60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441572de0, 0xc446c1b160, 0x0, 0xc42fb10068, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441572de0, 0xa74a80, 0xc446c1b160, 0x7fad6035c8e0, 0xc441572de0, 0x1038500, 0xc441572db0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc441572db0, 0xa74a80, 0xc446c1b160, 0xc4445aba40, 0xc4445aba60, 0x7fad6035c8e0, 0xc441572db0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441572db0, 0xa74a80, 0xc446c1b160, 0x7fad6035c8e0, 0xc441572db0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc441572db0, 0xa74a80, 0xc446c1b160, 0xc4445aba40, 0xc4445aba80, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xffffffff, 0x0, 0x19, 0xc439fbb7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xc4445aba40, 0xc4445abaa0, 0xc439fbb878, 0x558012, 0x588a5999, 0x3994a188)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xed01c5099, 0x0, 0x0, 0x4c5e6)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xc4445aba40, 0xc4445abac0, 0xa91b40, 0xc4445abb00, 0xc4445aba40, 0xc4445abb00)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xc420246100, 0xc42fb46c20, 0xc439fbba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xc4445aba40, 0xc4445aba60, 0x2c05, 0xc440d62b18, 0x0, 0xc440d62b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4415721e0, 0xc4430930e0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f20a56f)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2200, 0xc420331380, 0x10265c0, 0xc441572210, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2200, 0xc441572210)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9106 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43b08ef60, 0xc43b08ef90, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43b08ef60, 0xc441178c20, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43b08ef60, 0xc447e1b140, 0x0, 0xc440f5f090, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43b08ef60, 0xa74a80, 0xc447e1b140, 0x7fad6035c8e0, 0xc43b08ef60, 0x1038500, 0xc43b08ef30)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43b08ef30, 0xa74a80, 0xc447e1b140, 0xc441178b20, 0xc441178b40, 0x7fad6035c8e0, 0xc43b08ef30, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b08ef30, 0xa74a80, 0xc447e1b140, 0x7fad6035c8e0, 0xc43b08ef30, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43b08ef30, 0xa74a80, 0xc447e1b140, 0xc441178b20, 0xc441178b60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xffffffff, 0x0, 0x19, 0xc446a2d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xc441178b20, 0xc441178b80, 0xc446a2d878, 0x558012, 0x588a5813, 0x38229eae)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xed01c4f13, 0x0, 0x0, 0x5a1fa)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xc441178b20, 0xc441178ba0, 0xa91b40, 0xc441178bc0, 0xc441178b20, 0xc441178bc0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xc420246100, 0xc441a50070, 0xc446a2da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xc441178b20, 0xc441178b40, 0x290d, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43d26de90, 0xc4431f7e60, 0xc420179d60, 0x0, 0x0, 0xc420236800, 0x450280)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b4e00, 0xc420331380, 0x10265c0, 0xc43d26dec0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b4e00, 0xc43d26dec0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b4e00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 960 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048eb04)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048eb00)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x8b63b2637083582)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a336ff0, 0xc420162418, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a336c00, 0xc42a336ff0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a336c00, 0xc4240defa0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a336c00, 0xc4241f0d60, 0x0, 0xc42a33c970, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a336c00, 0xa74a80, 0xc4241f0d60, 0x7fad6035c8e0, 0xc42a336c00, 0x1038500, 0xc42a336bd0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a336bd0, 0xa74a80, 0xc4241f0d60, 0xc4240dea80, 0xc4240deaa0, 0x7fad6035c8e0, 0xc42a336bd0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a336bd0, 0xa74a80, 0xc4241f0d60, 0x7fad6035c8e0, 0xc42a336bd0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a336bd0, 0xa74a80, 0xc4241f0d60, 0xc4240dea80, 0xc4240deac0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xffffffff, 0x0, 0x19, 0xc4215dd7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xc4240dea80, 0xc4240deae0, 0xc4215dd878, 0x558012, 0x588a5741, 0x378ba2ff)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xed01c4e41, 0x0, 0x0, 0x1633d4)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xc4240dea80, 0xc4240deb00, 0xa91b40, 0xc4240deb40, 0xc4240dea80, 0xc4240deb40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xc420246100, 0xc42a2a6aa0, 0xc4215dda70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xc4240dea80, 0xc4240deaa0, 0xbe4a, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a077f20, 0xc42a1fed80, 0xc420179d60, 0x0, 0x0, 0x312d70691c126564, 0x2d312d30322d3237)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1500, 0xc420331380, 0x10265c0, 0xc42a077f50, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1500, 0xc42a077f50)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1500)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 1013 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048e0c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048e0c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x59362661c4be275c)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a1598f0, 0xc422334c2d, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a159560, 0xc42a1598f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a159560, 0xc424576380, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a159560, 0xc425337d40, 0x0, 0xc42a15c850, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a159560, 0xa74a80, 0xc425337d40, 0x7fad6035c8e0, 0xc42a159560, 0x1038500, 0xc42a159530)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a159530, 0xa74a80, 0xc425337d40, 0xc424581d00, 0xc424581d20, 0x7fad6035c8e0, 0xc42a159530, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a159530, 0xa74a80, 0xc425337d40, 0x7fad6035c8e0, 0xc42a159530, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a159530, 0xa74a80, 0xc425337d40, 0xc424581d00, 0xc424581d60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xffffffff, 0x0, 0x19, 0xc42005b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xc424581d00, 0xc424581d80, 0xc42005b878, 0x558012, 0x588a5741, 0x36275cc8)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xed01c4e41, 0x0, 0x0, 0x634c03)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xc424581d00, 0xc424581da0, 0xa91b40, 0xc424581de0, 0xc424581d00, 0xc424581de0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xc420246100, 0xc429eba670, 0xc42005ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xc424581d00, 0xc424581d20, 0x6671, 0xc427ba0b18, 0x0, 0xc427ba0b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a066510, 0xc428821260, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4204ff000, 0xc420331380, 0x10265c0, 0xc42a066540, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4204ff000, 0xc42a066540)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4204ff000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8818 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc44930ea6c)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*WaitGroup).Wait(0xc44930ea60)\n\t/usr/local/go/src/sync/waitgroup.go:131 +0x97\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc44930ea60, 0xc43cdd9380)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e\n\ngoroutine 8420 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43ffd7b60, 0xc43ffd7b90, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43ffd7b60, 0xc432956340, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43ffd7b60, 0xc43297ad60, 0x0, 0xc43c974718, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43ffd7b60, 0xa74a80, 0xc43297ad60, 0x7fad6035c8e0, 0xc43ffd7b60, 0x1038500, 0xc43ffd7b30)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43ffd7b30, 0xa74a80, 0xc43297ad60, 0xc4329561e0, 0xc432956220, 0x7fad6035c8e0, 0xc43ffd7b30, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43ffd7b30, 0xa74a80, 0xc43297ad60, 0x7fad6035c8e0, 0xc43ffd7b30, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43ffd7b30, 0xa74a80, 0xc43297ad60, 0xc4329561e0, 0xc432956260, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xffffffff, 0x0, 0x19, 0xc42ee537e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xc4329561e0, 0xc432956280, 0xc42ee53878, 0x558012, 0x588a57d4, 0xe57917e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xed01c4ed4, 0x0, 0x0, 0xbf481)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xc4329561e0, 0xc4329562a0, 0xa91b40, 0xc4329562c0, 0xc4329561e0, 0xc4329562c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xc420246100, 0xc4495f2940, 0xc42ee53a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xc4329561e0, 0xc432956220, 0x530e, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43b859230, 0xc44661b080, 0xc420179d60, 0x0, 0x0, 0xc43b43a600, 0xb)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421002000, 0xc420331380, 0x10265c0, 0xc43b859260, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421002000, 0xc43b859260)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421002000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 1286 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204962c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204962c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d96b7e0, 0x2, 0x2, 0xc42d711448, 0x0, 0xc42db9fa00)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d96d3b0, 0x159dc604330, 0x159dc64d710, 0xc42d96b7e0, 0x2, 0x2, 0xc42d96d300, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d96d3b0, 0xc42d96d2c0, 0x0, 0xc42d96b7b8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d96d3b0, 0xa934c0, 0xc42d96d2c0, 0x7fad6035c8e0, 0xc42d96d3b0, 0x1038500, 0xc42d96d380)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d96d380, 0xa934c0, 0xc42d96d2c0, 0xc42dbb7fc0, 0xc42dbb7fe0, 0x7fad6035c8e0, 0xc42d96d380, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d96d380, 0xa934c0, 0xc42d96d2c0, 0x7fad6035c8e0, 0xc42d96d380, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d96d380, 0xa934c0, 0xc42d96d2c0, 0xc42dbb7fc0, 0xc42dc4e000, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xffffffff, 0x0, 0x19, 0xc42d7117e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xc42dbb7fc0, 0xc42dc4e020, 0xc42d711878, 0x558012, 0x588a5743, 0x2a4230cc)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xc42d711900, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xc42dbb7fc0, 0xc42dc4e040, 0xa91b40, 0xc42dc4e060, 0xc42dbb7fc0, 0xc42dc4e060)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xc420246100, 0xc42d96b770, 0xc42d711a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xc42dbb7fc0, 0xc42dbb7fe0, 0x4e, 0xc42580fb18, 0x0, 0xc42580fb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d96d1a0, 0xc42d5aa780, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d055300, 0xc420331380, 0x10265d8, 0xc42d96d260, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d055300, 0xc42d96d260)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d055300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8354 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42cbe4810, 0xc42cbe4840, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42cbe4810, 0xc43ddc4e00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42cbe4810, 0xc4379f5e60, 0x0, 0xc42cb08288, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42cbe4810, 0xa74a80, 0xc4379f5e60, 0x7fad6035c8e0, 0xc42cbe4810, 0x1038500, 0xc42cbe47e0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42cbe47e0, 0xa74a80, 0xc4379f5e60, 0xc43ddc4d00, 0xc43ddc4d20, 0x7fad6035c8e0, 0xc42cbe47e0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cbe47e0, 0xa74a80, 0xc4379f5e60, 0x7fad6035c8e0, 0xc42cbe47e0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42cbe47e0, 0xa74a80, 0xc4379f5e60, 0xc43ddc4d00, 0xc43ddc4d40, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xffffffff, 0x0, 0x19, 0xc439edb7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xc43ddc4d00, 0xc43ddc4d60, 0xc439edb878, 0x558012, 0x588a579b, 0x3850790e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xed01c4e9b, 0x0, 0x0, 0xd358d)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xc43ddc4d00, 0xc43ddc4d80, 0xa91b40, 0xc43ddc4da0, 0xc43ddc4d00, 0xc43ddc4da0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xc420246100, 0xc42ce488f0, 0xc439edba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xc43ddc4d00, 0xc43ddc4d20, 0x9bf6, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42ce53e90, 0xc42ba4dbc0, 0xc420179d60, 0x0, 0x0, 0x637573070a0f0a63, 0x7274041273736563)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad300, 0xc420331380, 0x10265c0, 0xc42ce53ec0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad300, 0xc42ce53ec0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9896 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4387addec)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*WaitGroup).Wait(0xc4387adde0)\n\t/usr/local/go/src/sync/waitgroup.go:131 +0x97\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc4387adde0, 0xc44346c7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e\n\ngoroutine 956 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048eb04)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048eb00)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x2f2591367954edce)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc429d49e30, 0xc420162418, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc429d491d0, 0xc429d49e30, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc429d491d0, 0xc4257999c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc429d491d0, 0xc4258dc560, 0x0, 0xc429d4eed8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc429d491d0, 0xa74a80, 0xc4258dc560, 0x7fad6035c8e0, 0xc429d491d0, 0x1038500, 0xc429d491a0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc429d491a0, 0xa74a80, 0xc4258dc560, 0xc425798e20, 0xc425798e80, 0x7fad6035c8e0, 0xc429d491a0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429d491a0, 0xa74a80, 0xc4258dc560, 0x7fad6035c8e0, 0xc429d491a0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc429d491a0, 0xa74a80, 0xc4258dc560, 0xc425798e20, 0xc425798ec0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xffffffff, 0x0, 0x19, 0xc4294737e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xc425798e20, 0xc425798ee0, 0xc429473878, 0x558012, 0x588a5741, 0x32ff0d58)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xed01c4e41, 0x0, 0x0, 0x126687)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xc425798e20, 0xc425798f00, 0xa91b40, 0xc425798f20, 0xc425798e20, 0xc425798f20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xc420246100, 0xc429cb7770, 0xc429473a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xc425798e20, 0xc425798e80, 0xa86c, 0xc42023db18, 0x0, 0xc42023db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc429ccc150, 0xc427a01e60, 0xc420179d60, 0x0, 0x0, 0x312d70691c126564, 0x2d312d30322d3237)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed0800, 0xc420331380, 0x10265c0, 0xc429ccc180, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed0800, 0xc429ccc180)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed0800)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8565 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44293d9e0, 0xc44293da10, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44293d9e0, 0xc43fabcb20, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44293d9e0, 0xc43fab6a20, 0x0, 0xc429cfde08, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44293d9e0, 0xa74a80, 0xc43fab6a20, 0x7fad6035c8e0, 0xc44293d9e0, 0x1038500, 0xc44293d9b0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44293d9b0, 0xa74a80, 0xc43fab6a20, 0xc43fabc960, 0xc43fabca40, 0x7fad6035c8e0, 0xc44293d9b0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44293d9b0, 0xa74a80, 0xc43fab6a20, 0x7fad6035c8e0, 0xc44293d9b0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44293d9b0, 0xa74a80, 0xc43fab6a20, 0xc43fabc960, 0xc43fabca60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xffffffff, 0x0, 0x19, 0xc438c977e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xc43fabc960, 0xc43fabca80, 0xc438c97878, 0x558012, 0x588a59b7, 0x379b16b1)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xed01c50b7, 0x0, 0x0, 0x3bb05)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xc43fabc960, 0xc43fabcaa0, 0xa91b40, 0xc43fabcac0, 0xc43fabc960, 0xc43fabcac0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xc420246100, 0xc429d26b50, 0xc438c97a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xc43fabc960, 0xc43fabca40, 0x1571, 0xc420e1db18, 0x0, 0xc420e1db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44293cf30, 0xc43ead84e0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc433f6ee00, 0xc420331380, 0x10265c0, 0xc44293cf60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc433f6ee00, 0xc44293cf60)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc433f6ee00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8550 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc422c45b60, 0xc422c45b90, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc422c45b60, 0xc433bd8b40, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc422c45b60, 0xc433c797c0, 0x0, 0xc422b51c28, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc422c45b60, 0xa74a80, 0xc433c797c0, 0x7fad6035c8e0, 0xc422c45b60, 0x1038500, 0xc422c45b30)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc422c45b30, 0xa74a80, 0xc433c797c0, 0xc433bd89c0, 0xc433bd89e0, 0x7fad6035c8e0, 0xc422c45b30, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc422c45b30, 0xa74a80, 0xc433c797c0, 0x7fad6035c8e0, 0xc422c45b30, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc422c45b30, 0xa74a80, 0xc433c797c0, 0xc433bd89c0, 0xc433bd8a00, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xffffffff, 0x0, 0x19, 0xc448c337e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xc433bd89c0, 0xc433bd8a20, 0xc448c33878, 0x558012, 0x588a57b9, 0x390dce08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xed01c4eb9, 0x0, 0x0, 0x45cf1a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xc433bd89c0, 0xc433bd8a40, 0xa91b40, 0xc433bd8aa0, 0xc433bd89c0, 0xc433bd8aa0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xc420246100, 0xc422165600, 0xc448c33a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xc433bd89c0, 0xc433bd89e0, 0xbb32, 0xc420047b18, 0x0, 0xc420047b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4241a7a70, 0xc4235d56e0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f006322)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc425cf3c00, 0xc420331380, 0x10265c0, 0xc4241a7aa0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc425cf3c00, 0xc4241a7aa0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc425cf3c00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9963 [chan receive, 8 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc446610870, 0xc435e3bc20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47\nnet/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc446610870, 0xc435e3bc20)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc446610870, 0xc435e3bc20)\n\t/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc4417fa480, 0xc435e3bc20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab\nnet/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc4417fa480, 0xc435e3bc20)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc4417fa480, 0xc435e3bc20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc4417fa440, 0xc435e3ba40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2\nnet/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc4417fa440, 0xc435e3ba40)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc4417fa420, 0xc435e3ba40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c\nnet/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc4417fa420, 0xc435e3ba40)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc42bd16b60, 0xc435e3ba40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449\nnet/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc42bd16b60, 0xc435e3b950)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc42bd16b60, 0xc435e3b950)\n\t/usr/local/go/src/net/http/server.go:2202 +0x7d\nnet/http.(*conn).serve(0xc43269b480, 0x1038440, 0xc43ed8f1c0)\n\t/usr/local/go/src/net/http/server.go:1579 +0x4b7\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:2293 +0x44d\n\ngoroutine 5310 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048f704)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048f700)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x2aa3be3b56d99d3e)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc448cff200, 0xc420162418, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc448cfe9c0, 0xc448cff200, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc448cfe9c0, 0xc42b93a900, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc448cfe9c0, 0xc42b9994a0, 0x0, 0xc448c5e650, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc448cfe9c0, 0xa74a80, 0xc42b9994a0, 0x7fad6035c8e0, 0xc448cfe9c0, 0x1038500, 0xc448cfe990)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc448cfe990, 0xa74a80, 0xc42b9994a0, 0xc42b93fd20, 0xc42b93fd60, 0x7fad6035c8e0, 0xc448cfe990, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc448cfe990, 0xa74a80, 0xc42b9994a0, 0x7fad6035c8e0, 0xc448cfe990, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc448cfe990, 0xa74a80, 0xc42b9994a0, 0xc42b93fd20, 0xc42b93fda0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xffffffff, 0x0, 0x19, 0xc43ee3d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xc42b93fd20, 0xc42b93fdc0, 0xc43ee3d878, 0x558012, 0x588a575f, 0x32fe05f8)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xed01c4e5f, 0x0, 0x0, 0x15eca2)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xc42b93fd20, 0xc42b93fde0, 0xa91b40, 0xc42b93fe20, 0xc42b93fd20, 0xc42b93fe20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xc420246100, 0xc4493beae0, 0xc43ee3da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xc42b93fd20, 0xc42b93fd60, 0xae8b, 0xc42023db18, 0x0, 0xc42023db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4493ca6c0, 0xc43eb0d560, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421003200, 0xc420331380, 0x10265c0, 0xc4493ca6f0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421003200, 0xc4493ca6f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421003200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8149 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d4f39b0, 0xc42d4f39e0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d4f39b0, 0xc437acb8c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d4f39b0, 0xc437af3980, 0x0, 0xc42d4e08c0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d4f39b0, 0xa74a80, 0xc437af3980, 0x7fad6035c8e0, 0xc42d4f39b0, 0x1038500, 0xc42d4f3980)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d4f3980, 0xa74a80, 0xc437af3980, 0xc437acb780, 0xc437acb7a0, 0x7fad6035c8e0, 0xc42d4f3980, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d4f3980, 0xa74a80, 0xc437af3980, 0x7fad6035c8e0, 0xc42d4f3980, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d4f3980, 0xa74a80, 0xc437af3980, 0xc437acb780, 0xc437acb7e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xffffffff, 0x0, 0x19, 0xc439edd7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xc437acb780, 0xc437acb800, 0xc439edd878, 0x558012, 0x588a579b, 0x36ace23c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xc439edd900, 0x0, 0x0, 0xbf)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xc437acb780, 0xc437acb820, 0xa91b40, 0xc437acb860, 0xc437acb780, 0xc437acb860)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xc420246100, 0xc427518680, 0xc439edda70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xc437acb780, 0xc437acb7a0, 0xa5f8, 0xc42023cb18, 0x0, 0xc42023cb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc428ee6120, 0xc44088f2c0, 0xc420179d60, 0x0, 0x0, 0x6c616e7265746e69, 0x412656c020a0a0a)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c300, 0xc420331380, 0x10265c0, 0xc428ee61e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c300, 0xc428ee61e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 11110 [semacquire, 6 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc43de094a0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc434eb1b10, 0xc43de094a0, 0x10343c0, 0xc4201e3040)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326\n\ngoroutine 8324 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42e393d10, 0xc42e393d40, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42e393d10, 0xc437c30680, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42e393d10, 0xc437cbb800, 0x0, 0xc42a5bac10, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42e393d10, 0xa74a80, 0xc437cbb800, 0x7fad6035c8e0, 0xc42e393d10, 0x1038500, 0xc42e393ce0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42e393ce0, 0xa74a80, 0xc437cbb800, 0xc437c30540, 0xc437c30560, 0x7fad6035c8e0, 0xc42e393ce0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e393ce0, 0xa74a80, 0xc437cbb800, 0x7fad6035c8e0, 0xc42e393ce0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42e393ce0, 0xa74a80, 0xc437cbb800, 0xc437c30540, 0xc437c30580, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0xffffffff, 0x0, 0x19, 0xc4464b37e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0xc437c30540, 0xc437c305a0, 0xc4464b3878, 0x558012, 0x588a5798, 0xdc5a87c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0x411715, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0xc437c30540, 0xc437c305c0, 0xa91b40, 0xc437c30600, 0xc437c30540, 0xc437c30600)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0xc420246100, 0xc42a685c90, 0xc4464b3a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0xc437c30540, 0xc437c30560, 0x6b39, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4248f1b90, 0xc42ba4c120, 0xc420179d60, 0x0, 0x0, 0x646e696b040a100a, 0x61646174656d0812)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3200, 0xc420331380, 0x10265c0, 0xc4267f4030, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3200, 0xc4267f4030)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8551 [chan receive, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc42443cd20, 0xc443d482d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47\nnet/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc42443cd20, 0xc443d482d0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc42443cd20, 0xc443d482d0)\n\t/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc433a8a760, 0xc443d482d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab\nnet/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc433a8a760, 0xc443d482d0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc433a8a760, 0xc443d482d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc433a8a740, 0xc443d480f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2\nnet/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc433a8a740, 0xc443d480f0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc433a8a6e0, 0xc443d480f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c\nnet/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc433a8a6e0, 0xc443d480f0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4292dfba0, 0xc443d480f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449\nnet/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4292dfba0, 0xc443d48000)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4292dfba0, 0xc443d48000)\n\t/usr/local/go/src/net/http/server.go:2202 +0x7d\nnet/http.(*conn).serve(0xc43a2a0a80, 0x1038440, 0xc427f107c0)\n\t/usr/local/go/src/net/http/server.go:1579 +0x4b7\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:2293 +0x44d\n\ngoroutine 8224 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42e2649c0, 0xc42e2649f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42e2649c0, 0xc437c38700, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42e2649c0, 0xc437c85c40, 0x0, 0xc42a5b8838, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42e2649c0, 0xa74a80, 0xc437c85c40, 0x7fad6035c8e0, 0xc42e2649c0, 0x1038500, 0xc42e264990)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42e264990, 0xa74a80, 0xc437c85c40, 0xc437c38580, 0xc437c38600, 0x7fad6035c8e0, 0xc42e264990, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e264990, 0xa74a80, 0xc437c85c40, 0x7fad6035c8e0, 0xc42e264990, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42e264990, 0xa74a80, 0xc437c85c40, 0xc437c38580, 0xc437c38620, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xffffffff, 0x0, 0x19, 0xc4454c97e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xc437c38580, 0xc437c38640, 0xc4454c9878, 0x558012, 0x588a5798, 0xdc3489f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xed01c4e98, 0x0, 0x0, 0x172211)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xc437c38580, 0xc437c38660, 0xa91b40, 0xc437c38680, 0xc437c38580, 0xc437c38680)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xc420246100, 0xc42bc95f40, 0xc4454c9a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xc437c38580, 0xc437c38600, 0x4809, 0xc420e1db18, 0x0, 0xc420e1db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc426b9e150, 0xc4296faea0, 0xc420179d60, 0x0, 0x0, 0xc43ceb47a8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4275e7c00, 0xc420331380, 0x10265c0, 0xc426b9e180, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4275e7c00, 0xc426b9e180)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4275e7c00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9093 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc443740930, 0xc4437409f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc443740930, 0xc43cae2ec0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc443740930, 0xc438b40d20, 0x0, 0xc447b12208, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc443740930, 0xa74a80, 0xc438b40d20, 0x7fad6035c8e0, 0xc443740930, 0x1038500, 0xc4437408d0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4437408d0, 0xa74a80, 0xc438b40d20, 0xc43d181fa0, 0xc43cae2080, 0x7fad6035c8e0, 0xc4437408d0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4437408d0, 0xa74a80, 0xc438b40d20, 0x7fad6035c8e0, 0xc4437408d0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4437408d0, 0xa74a80, 0xc438b40d20, 0xc43d181fa0, 0xc43cae2280, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xffffffff, 0x0, 0x19, 0xc444c017e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xc43d181fa0, 0xc43cae2540, 0xc444c01878, 0x558012, 0x588a5810, 0xbca599a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xc444c01900, 0x0, 0x0, 0x4d)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xc43d181fa0, 0xc43cae2760, 0xa91b40, 0xc43cae29a0, 0xc43d181fa0, 0xc43cae29a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xc420246100, 0xc43e19f190, 0xc444c01a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xc43d181fa0, 0xc43cae2080, 0x7e56, 0xc42023bb18, 0x0, 0xc42023bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc444f08630, 0xc449d8a420, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a6d00, 0xc420331380, 0x10265c0, 0xc444f08870, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a6d00, 0xc444f08870)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a6d00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8564 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43c8cbcb0, 0xc43c8cbce0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43c8cbcb0, 0xc43bba7d20, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43c8cbcb0, 0xc44092a560, 0x0, 0xc4272a5c18, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43c8cbcb0, 0xa74a80, 0xc44092a560, 0x7fad6035c8e0, 0xc43c8cbcb0, 0x1038500, 0xc43c8cbc80)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43c8cbc80, 0xa74a80, 0xc44092a560, 0xc43bba7c20, 0xc43bba7c40, 0x7fad6035c8e0, 0xc43c8cbc80, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c8cbc80, 0xa74a80, 0xc44092a560, 0x7fad6035c8e0, 0xc43c8cbc80, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43c8cbc80, 0xa74a80, 0xc44092a560, 0xc43bba7c20, 0xc43bba7c60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xffffffff, 0x0, 0x19, 0xc438c957e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xc43bba7c20, 0xc43bba7c80, 0xc438c95878, 0x558012, 0x588a59b7, 0x3416450e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xed01c50b7, 0x0, 0x0, 0x1559c1)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xc43bba7c20, 0xc43bba7ca0, 0xa91b40, 0xc43bba7cc0, 0xc43bba7c20, 0xc43bba7cc0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xc420246100, 0xc42d842630, 0xc438c95a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xc43bba7c20, 0xc43bba7c40, 0x4558, 0xc420e1db18, 0x0, 0xc420e1db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43c14bf50, 0xc43ead8300, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6eb00, 0xc420331380, 0x10265c0, 0xc43c14bf80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6eb00, 0xc43c14bf80)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6eb00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 11987 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d1ba570, 0xc42d1ba630, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1ba570, 0xc44320c400, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1ba570, 0xc444b165a0, 0x0, 0xc42fe8a450, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d1ba570, 0xa74a80, 0xc444b165a0, 0x7fad6035c8e0, 0xc42d1ba570, 0x1038500, 0xc42d1ba4b0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d1ba4b0, 0xa74a80, 0xc444b165a0, 0xc44320c300, 0xc44320c320, 0x7fad6035c8e0, 0xc42d1ba4b0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1ba4b0, 0xa74a80, 0xc444b165a0, 0x7fad6035c8e0, 0xc42d1ba4b0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d1ba4b0, 0xa74a80, 0xc444b165a0, 0xc44320c300, 0xc44320c340, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0xffffffff, 0x0, 0x19, 0xc4377097e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0xc44320c300, 0xc44320c360, 0xc437709878, 0x558012, 0x588a59d5, 0x3832a21c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0x41167e, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0xc44320c300, 0xc44320c380, 0xa91b40, 0xc44320c3a0, 0xc44320c300, 0xc44320c3a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0xc420246100, 0xc430f2b1f0, 0xc437709a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0xc44320c300, 0xc44320c320, 0x4a1b, 0xc440d64b18, 0x0, 0xc440d64b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d3a5260, 0xc44103c960, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2100, 0xc420331380, 0x10265c0, 0xc42d3a5f20, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2100, 0xc42d3a5f20)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2100)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8863 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc449db7cb0, 0xc449db7ce0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc449db7cb0, 0xc44a5053e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc449db7cb0, 0xc43e3324c0, 0x0, 0xc445048050, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc449db7cb0, 0xa74a80, 0xc43e3324c0, 0x7fad6035c8e0, 0xc449db7cb0, 0x1038500, 0xc449db7c80)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc449db7c80, 0xa74a80, 0xc43e3324c0, 0xc44a5052e0, 0xc44a505300, 0x7fad6035c8e0, 0xc449db7c80, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449db7c80, 0xa74a80, 0xc43e3324c0, 0x7fad6035c8e0, 0xc449db7c80, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc449db7c80, 0xa74a80, 0xc43e3324c0, 0xc44a5052e0, 0xc44a505320, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xffffffff, 0x0, 0x19, 0xc43ed837e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xc44a5052e0, 0xc44a505340, 0xc43ed83878, 0x558012, 0x588a57f5, 0x371d60e2)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xed01c4ef5, 0x0, 0x0, 0xafff3)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xc44a5052e0, 0xc44a505360, 0xa91b40, 0xc44a505380, 0xc44a5052e0, 0xc44a505380)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xc420246100, 0xc426faf720, 0xc43ed83a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xc44a5052e0, 0xc44a505300, 0x4ef8, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43ac0fd70, 0xc42c213aa0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f1a9759)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87c00, 0xc420331380, 0x10265c0, 0xc43ac0fda0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87c00, 0xc43ac0fda0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87c00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8439 [chan send, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc42c050d20, 0xc429a59a40, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 8328 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d30f5f0, 0xc42d30f620, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d30f5f0, 0xc437a69900, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d30f5f0, 0xc437c31820, 0x0, 0xc42d311020, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d30f5f0, 0xa74a80, 0xc437c31820, 0x7fad6035c8e0, 0xc42d30f5f0, 0x1038500, 0xc42d30f5c0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d30f5c0, 0xa74a80, 0xc437c31820, 0xc437a697e0, 0xc437a69820, 0x7fad6035c8e0, 0xc42d30f5c0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d30f5c0, 0xa74a80, 0xc437c31820, 0x7fad6035c8e0, 0xc42d30f5c0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d30f5c0, 0xa74a80, 0xc437c31820, 0xc437a697e0, 0xc437a69840, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xffffffff, 0x0, 0x19, 0xc4464b57e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xc437a697e0, 0xc437a69860, 0xc4464b5878, 0x558012, 0x588a579b, 0x36b5bea6)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xed01c4e9b, 0x0, 0x0, 0xe9a86)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xc437a697e0, 0xc437a69880, 0xa91b40, 0xc437a698a0, 0xc437a697e0, 0xc437a698a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xc420246100, 0xc42a5bacd0, 0xc4464b5a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xc437a697e0, 0xc437a69820, 0xa56e, 0xc420049b18, 0x0, 0xc420049b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc426bbc0f0, 0xc42ba4c960, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516500, 0xc420331380, 0x10265c0, 0xc426bbc120, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516500, 0xc426bbc120)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516500)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8072 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4244583c0, 0xc4244583f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4244583c0, 0xc433ff26a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4244583c0, 0xc4340f8140, 0x0, 0xc425ba8a88, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4244583c0, 0xa74a80, 0xc4340f8140, 0x7fad6035c8e0, 0xc4244583c0, 0x1038500, 0xc424458390)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc424458390, 0xa74a80, 0xc4340f8140, 0xc433ff2540, 0xc433ff2560, 0x7fad6035c8e0, 0xc424458390, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc424458390, 0xa74a80, 0xc4340f8140, 0x7fad6035c8e0, 0xc424458390, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc424458390, 0xa74a80, 0xc4340f8140, 0xc433ff2540, 0xc433ff2580, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xffffffff, 0x0, 0x19, 0xc43b1b57e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xc433ff2540, 0xc433ff25a0, 0xc43b1b5878, 0x558012, 0x588a57b6, 0xbf5352e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xed01c4eb6, 0x0, 0x0, 0xe71e4)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xc433ff2540, 0xc433ff25e0, 0xa91b40, 0xc433ff2620, 0xc433ff2540, 0xc433ff2620)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xc420246100, 0xc4278f60a0, 0xc43b1b5a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xc433ff2540, 0xc433ff2560, 0x73d9, 0xc42023cb18, 0x0, 0xc42023cb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc427c9a210, 0xc42779e0c0, 0xc420179d60, 0x0, 0x0, 0x4d1279726f746973, 0x6f642e7865646e69)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bc000, 0xc420331380, 0x10265c0, 0xc427c9a240, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bc000, 0xc427c9a240)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bc000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10350 [chan send, 8 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc43aa882d0, 0xc427521aa0, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 11427 [chan receive, 4 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc437232960, 0xc441ff1b30)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47\nnet/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc437232960, 0xc441ff1b30)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc437232960, 0xc441ff1b30)\n\t/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc43d66fc80, 0xc441ff1b30)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab\nnet/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc43d66fc80, 0xc441ff1b30)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc43d66fc80, 0xc441ff1b30)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc43d66fc40, 0xc441ff1950)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2\nnet/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc43d66fc40, 0xc441ff1950)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc43d66fc20, 0xc441ff1950)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c\nnet/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc43d66fc20, 0xc441ff1950)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc42bd16270, 0xc441ff1950)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449\nnet/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc42bd16270, 0xc441ff1860)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc42bd16270, 0xc441ff1860)\n\t/usr/local/go/src/net/http/server.go:2202 +0x7d\nnet/http.(*conn).serve(0xc42ce81e80, 0x1038440, 0xc4391235c0)\n\t/usr/local/go/src/net/http/server.go:1579 +0x4b7\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:2293 +0x44d\n\ngoroutine 11217 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42e09da10, 0xc42e09da70, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42e09da10, 0xc44448ba60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42e09da10, 0xc43a130000, 0x0, 0xc4351d5b50, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42e09da10, 0xa74a80, 0xc43a130000, 0x7fad6035c8e0, 0xc42e09da10, 0x1038500, 0xc42e09d9b0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42e09d9b0, 0xa74a80, 0xc43a130000, 0xc44448b960, 0xc44448b980, 0x7fad6035c8e0, 0xc42e09d9b0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e09d9b0, 0xa74a80, 0xc43a130000, 0x7fad6035c8e0, 0xc42e09d9b0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42e09d9b0, 0xa74a80, 0xc43a130000, 0xc44448b960, 0xc44448b9a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xffffffff, 0x0, 0x19, 0xc4491b17e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xc44448b960, 0xc44448b9c0, 0xc4491b1878, 0x558012, 0x588a595d, 0x3aab877f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xed01c505d, 0x0, 0x0, 0xbac3b)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xc44448b960, 0xc44448b9e0, 0xa91b40, 0xc44448ba00, 0xc44448b960, 0xc44448ba00)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xc420246100, 0xc435236050, 0xc4491b1a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xc44448b960, 0xc44448b980, 0x74a2, 0xc440d64b18, 0x0, 0xc440d64b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42e4228a0, 0xc448a02000, 0xc420179d60, 0x0, 0x0, 0xc434e02fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a7d00, 0xc420331380, 0x10265c0, 0xc42e422900, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a7d00, 0xc42e422900)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a7d00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8701 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc431022600, 0xc431022630, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc431022600, 0xc4327302a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc431022600, 0xc4328d87c0, 0x0, 0xc4310247d8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc431022600, 0xa74a80, 0xc4328d87c0, 0x7fad6035c8e0, 0xc431022600, 0x1038500, 0xc4310225d0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4310225d0, 0xa74a80, 0xc4328d87c0, 0xc432730100, 0xc432730120, 0x7fad6035c8e0, 0xc4310225d0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4310225d0, 0xa74a80, 0xc4328d87c0, 0x7fad6035c8e0, 0xc4310225d0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4310225d0, 0xa74a80, 0xc4328d87c0, 0xc432730100, 0xc432730180, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0xffffffff, 0x0, 0x19, 0xc42eba97e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0xc432730100, 0xc4327301a0, 0xc42eba9878, 0x558012, 0x588a57d7, 0x33ad55a1)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0x411715, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0xc432730100, 0xc4327301c0, 0xa91b40, 0xc432730220, 0xc432730100, 0xc432730220)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0xc420246100, 0xc4421e7910, 0xc42eba9a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0xc432730100, 0xc432730120, 0x7a5e, 0xc420313b18, 0x0, 0xc420313b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43acec270, 0xc441296900, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f69cfe7)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1a00, 0xc420331380, 0x10265c0, 0xc43acec2a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1a00, 0xc43acec2a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1a00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8566 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc440460690, 0xc4404606c0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc440460690, 0xc448f51aa0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc440460690, 0xc448f30b80, 0x0, 0xc42e5dd498, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc440460690, 0xa74a80, 0xc448f30b80, 0x7fad6035c8e0, 0xc440460690, 0x1038500, 0xc440460660)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc440460660, 0xa74a80, 0xc448f30b80, 0xc448f519a0, 0xc448f519c0, 0x7fad6035c8e0, 0xc440460660, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440460660, 0xa74a80, 0xc448f30b80, 0x7fad6035c8e0, 0xc440460660, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc440460660, 0xa74a80, 0xc448f30b80, 0xc448f519a0, 0xc448f519e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xffffffff, 0x0, 0x19, 0xc438c937e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xc448f519a0, 0xc448f51a00, 0xc438c93878, 0x558012, 0x588a59b7, 0x37e74cb5)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xed01c50b7, 0x0, 0x0, 0x92d84)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xc448f519a0, 0xc448f51a20, 0xa91b40, 0xc448f51a40, 0xc448f519a0, 0xc448f51a40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xc420246100, 0xc4288a5000, 0xc438c93a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xc448f519a0, 0xc448f519c0, 0x6564, 0xc420314b18, 0x0, 0xc420314b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4461b4750, 0xc43ead8780, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f300, 0xc420331380, 0x10265c0, 0xc4461b4780, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f300, 0xc4461b4780)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8146 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc447f6ddd0, 0xc447f6de00, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc447f6ddd0, 0xc438caf440, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc447f6ddd0, 0xc438d08060, 0x0, 0xc4443e8b00, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc447f6ddd0, 0xa74a80, 0xc438d08060, 0x7fad6035c8e0, 0xc447f6ddd0, 0x1038500, 0xc447f6dda0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc447f6dda0, 0xa74a80, 0xc438d08060, 0xc438caf320, 0xc438caf340, 0x7fad6035c8e0, 0xc447f6dda0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447f6dda0, 0xa74a80, 0xc438d08060, 0x7fad6035c8e0, 0xc447f6dda0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc447f6dda0, 0xa74a80, 0xc438d08060, 0xc438caf320, 0xc438caf360, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xffffffff, 0x0, 0x19, 0xc42d1f37e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xc438caf320, 0xc438caf380, 0xc42d1f3878, 0x558012, 0x588a577d, 0x33f8d325)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xed01c4e7d, 0x0, 0x0, 0x186123)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xc438caf320, 0xc438caf3a0, 0xa91b40, 0xc438caf3c0, 0xc438caf320, 0xc438caf3c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xc420246100, 0xc442adb0e0, 0xc42d1f3a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xc438caf320, 0xc438caf340, 0x8687, 0xc420044b18, 0x0, 0xc420044b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442346330, 0xc423b1fec0, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abe00, 0xc420331380, 0x10265c0, 0xc442346360, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abe00, 0xc442346360)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abe00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8987 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43f7318c0, 0xc43f7318f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43f7318c0, 0xc4481f6a00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43f7318c0, 0xc43dd56d00, 0x0, 0xc43fc7cdd0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43f7318c0, 0xa74a80, 0xc43dd56d00, 0x7fad6035c8e0, 0xc43f7318c0, 0x1038500, 0xc43f731890)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43f731890, 0xa74a80, 0xc43dd56d00, 0xc4481f68e0, 0xc4481f6900, 0x7fad6035c8e0, 0xc43f731890, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43f731890, 0xa74a80, 0xc43dd56d00, 0x7fad6035c8e0, 0xc43f731890, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43f731890, 0xa74a80, 0xc43dd56d00, 0xc4481f68e0, 0xc4481f6940, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xffffffff, 0x0, 0x19, 0xc44a48b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xc4481f68e0, 0xc4481f6960, 0xc44a48b878, 0x558012, 0x588a5813, 0x36eaff71)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xed01c4f13, 0x0, 0x0, 0x669e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xc4481f68e0, 0xc4481f6980, 0xa91b40, 0xc4481f69a0, 0xc4481f68e0, 0xc4481f69a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xc420246100, 0xc442d71730, 0xc44a48ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xc4481f68e0, 0xc4481f6900, 0x34ca, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc441afaed0, 0xc4431f7a40, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1a00, 0xc420331380, 0x10265c0, 0xc441afb2f0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1a00, 0xc441afb2f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1a00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8080 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc423781020, 0xc423781050, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc423781020, 0xc433d34460, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc423781020, 0xc433d80b80, 0x0, 0xc423602a10, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc423781020, 0xa74a80, 0xc433d80b80, 0x7fad6035c8e0, 0xc423781020, 0x1038500, 0xc423780ff0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc423780ff0, 0xa74a80, 0xc433d80b80, 0xc433d34280, 0xc433d342c0, 0x7fad6035c8e0, 0xc423780ff0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423780ff0, 0xa74a80, 0xc433d80b80, 0x7fad6035c8e0, 0xc423780ff0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc423780ff0, 0xa74a80, 0xc433d80b80, 0xc433d34280, 0xc433d342e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xffffffff, 0x0, 0x19, 0xc448f017e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xc433d34280, 0xc433d34320, 0xc448f01878, 0x558012, 0x588a57b9, 0x336997c9)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xed01c4eb9, 0x0, 0x0, 0xf35b5)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xc433d34280, 0xc433d34340, 0xa91b40, 0xc433d34380, 0xc433d34280, 0xc433d34380)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xc420246100, 0xc423b4dec0, 0xc448f01a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xc433d34280, 0xc433d342c0, 0x7594, 0xc420319b18, 0x0, 0xc420319b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4239e4a20, 0xc42779f8c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5d1149)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bcd00, 0xc420331380, 0x10265c0, 0xc4239e4a50, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bcd00, 0xc4239e4a50)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bcd00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 11290 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc429a53200, 0xc429a53320, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc429a53200, 0xc440d0f2a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc429a53200, 0xc43c0a5340, 0x0, 0xc432dd5cc0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc429a53200, 0xa74a80, 0xc43c0a5340, 0x7fad6035c8e0, 0xc429a53200, 0x1038500, 0xc429a53140)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc429a53140, 0xa74a80, 0xc43c0a5340, 0xc440d0f180, 0xc440d0f1a0, 0x7fad6035c8e0, 0xc429a53140, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429a53140, 0xa74a80, 0xc43c0a5340, 0x7fad6035c8e0, 0xc429a53140, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc429a53140, 0xa74a80, 0xc43c0a5340, 0xc440d0f180, 0xc440d0f1c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xffffffff, 0x0, 0x19, 0xc437c817e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xc440d0f180, 0xc440d0f1e0, 0xc437c81878, 0x558012, 0x588a5978, 0x12bd2347)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xed01c5078, 0x0, 0x0, 0xdd0ec0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xc440d0f180, 0xc440d0f220, 0xa91b40, 0xc440d0f240, 0xc440d0f180, 0xc440d0f240)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xc420246100, 0xc432fa2cf0, 0xc437c81a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xc440d0f180, 0xc440d0f1a0, 0x7885, 0xc420e18b18, 0x0, 0xc420e18b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a4a0e10, 0xc442bd4360, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa000, 0xc420331380, 0x10265c0, 0xc42a4a0fc0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa000, 0xc42a4a0fc0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 1207 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204962c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204962c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42ca1eb80, 0x3, 0x3, 0xc429471448, 0x0, 0xc42c9c1800)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42c9cf7a0, 0x159dc604330, 0x159dc64d710, 0xc42ca1eb80, 0x3, 0x3, 0xc42c9cf700, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42c9cf7a0, 0xc42c9cf680, 0x0, 0xc42c9cc4d8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42c9cf7a0, 0xa934c0, 0xc42c9cf680, 0x7fad6035c8e0, 0xc42c9cf7a0, 0x1038500, 0xc42c9cf770)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c9cf770, 0xa934c0, 0xc42c9cf680, 0xc42ca1ea80, 0xc42ca1eaa0, 0x7fad6035c8e0, 0xc42c9cf770, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9cf770, 0xa934c0, 0xc42c9cf680, 0x7fad6035c8e0, 0xc42c9cf770, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c9cf770, 0xa934c0, 0xc42c9cf680, 0xc42ca1ea80, 0xc42ca1eac0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xffffffff, 0x0, 0x19, 0xc4294717e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xc42ca1ea80, 0xc42ca1eae0, 0xc429471878, 0x558012, 0x588a5743, 0x853622e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xed01c4e43, 0x0, 0x0, 0xae05)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xc42ca1ea80, 0xc42ca1eb00, 0xa91b40, 0xc42ca1eb20, 0xc42ca1ea80, 0xc42ca1eb20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xc420246100, 0xc42c9cc480, 0xc429471a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xc42ca1ea80, 0xc42ca1eaa0, 0x65, 0xc420319b18, 0x0, 0xc420319b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42c9cf560, 0xc42ba39140, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429599c00, 0xc420331380, 0x10265d8, 0xc42c9cf620, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429599c00, 0xc42c9cf620)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429599c00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 12194 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43a487560, 0xc43a487590, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43a487560, 0xc43d7009e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43a487560, 0xc43e05fec0, 0x0, 0xc426d06b28, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43a487560, 0xa74a80, 0xc43e05fec0, 0x7fad6035c8e0, 0xc43a487560, 0x1038500, 0xc43a487530)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43a487530, 0xa74a80, 0xc43e05fec0, 0xc43d7008e0, 0xc43d700900, 0x7fad6035c8e0, 0xc43a487530, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43a487530, 0xa74a80, 0xc43e05fec0, 0x7fad6035c8e0, 0xc43a487530, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43a487530, 0xa74a80, 0xc43e05fec0, 0xc43d7008e0, 0xc43d700920, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0xffffffff, 0x0, 0x19, 0xc435d4b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0xc43d7008e0, 0xc43d700940, 0xc435d4b878, 0x558012, 0x588a59f3, 0x3b313cc2)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0x41167e, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0xc43d7008e0, 0xc43d700960, 0xa91b40, 0xc43d700980, 0xc43d7008e0, 0xc43d700980)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0xc420246100, 0xc427760030, 0xc435d4ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0xc43d7008e0, 0xc43d700900, 0x52ec, 0xc435e50b18, 0x0, 0xc435e50b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc449610a80, 0xc449598de0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065100, 0xc420331380, 0x10265c0, 0xc449610ab0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065100, 0xc449610ab0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065100)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 1233 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048bb84)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048bb80)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x476db0d6341b0a2)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d5080a0, 0x2, 0x2, 0xc429477448, 0x0, 0xc42d4f0200)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d506240, 0x159dc604330, 0x159dc64d710, 0xc42d5080a0, 0x2, 0x2, 0xc42d506200, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d506240, 0xc42d506150, 0x0, 0xc42d508078, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d506240, 0xa934c0, 0xc42d506150, 0x7fad6035c8e0, 0xc42d506240, 0x1038500, 0xc42d506210)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d506210, 0xa934c0, 0xc42d506150, 0xc42d4ee4e0, 0xc42d4ee500, 0x7fad6035c8e0, 0xc42d506210, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d506210, 0xa934c0, 0xc42d506150, 0x7fad6035c8e0, 0xc42d506210, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d506210, 0xa934c0, 0xc42d506150, 0xc42d4ee4e0, 0xc42d4ee520, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xffffffff, 0x0, 0x19, 0xc4294777e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xc42d4ee4e0, 0xc42d4ee540, 0xc429477878, 0x558012, 0x588a5743, 0x205750f8)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xed01c4e43, 0x0, 0x0, 0x5c59)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xc42d4ee4e0, 0xc42d4ee560, 0xa91b40, 0xc42d4ee580, 0xc42d4ee4e0, 0xc42d4ee580)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xc420246100, 0xc42d508030, 0xc429477a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xc42d4ee4e0, 0xc42d4ee500, 0x48, 0xc42023bb18, 0x0, 0xc42023bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d506030, 0xc42d260900, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41fa28d46)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504000, 0xc420331380, 0x10265d8, 0xc42d5060f0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504000, 0xc42d5060f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 1316 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc42048bb84)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc42048bb80)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x476db0d6341b0a2)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d7c82f0, 0x2, 0x2, 0xc4293df448, 0x0, 0xc42d6a1400)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d7864b0, 0x159dc604330, 0x159dc64d710, 0xc42d7c82f0, 0x2, 0x2, 0xc42d786400, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d7864b0, 0xc42d7863c0, 0x0, 0xc42d7c82c8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d7864b0, 0xa934c0, 0xc42d7863c0, 0x7fad6035c8e0, 0xc42d7864b0, 0x1038500, 0xc42d786480)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d786480, 0xa934c0, 0xc42d7863c0, 0xc42d73b260, 0xc42d73b280, 0x7fad6035c8e0, 0xc42d786480, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d786480, 0xa934c0, 0xc42d7863c0, 0x7fad6035c8e0, 0xc42d786480, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d786480, 0xa934c0, 0xc42d7863c0, 0xc42d73b260, 0xc42d73b2a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xffffffff, 0x0, 0x19, 0xc4293df7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xc42d73b260, 0xc42d73b2c0, 0xc4293df878, 0x558012, 0x588a5743, 0x2362a750)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xed01c4e43, 0x0, 0x0, 0x60c3)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xc42d73b260, 0xc42d73b2e0, 0xa91b40, 0xc42d73b300, 0xc42d73b260, 0xc42d73b300)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xc420246100, 0xc42d7c8280, 0xc4293dfa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xc42d73b260, 0xc42d73b280, 0x48, 0xc425812b18, 0x0, 0xc425812b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d7862a0, 0xc42d260c60, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41fa28d46)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504600, 0xc420331380, 0x10265d8, 0xc42d786360, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504600, 0xc42d786360)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8738 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc443ff594c)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*WaitGroup).Wait(0xc443ff5940)\n\t/usr/local/go/src/sync/waitgroup.go:131 +0x97\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc443ff5940, 0xc443c341e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e\n\ngoroutine 1258 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204962c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204962c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d351440, 0x2, 0x2, 0xc429109448, 0x0, 0xc42d386880)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d3a4690, 0x159dc604330, 0x159dc64d710, 0xc42d351440, 0x2, 0x2, 0xc42d3a4600, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d3a4690, 0xc42d3a45a0, 0x0, 0xc42d351418, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d3a4690, 0xa934c0, 0xc42d3a45a0, 0x7fad6035c8e0, 0xc42d3a4690, 0x1038500, 0xc42d3a4660)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d3a4660, 0xa934c0, 0xc42d3a45a0, 0xc42d377ca0, 0xc42d377cc0, 0x7fad6035c8e0, 0xc42d3a4660, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a4660, 0xa934c0, 0xc42d3a45a0, 0x7fad6035c8e0, 0xc42d3a4660, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d3a4660, 0xa934c0, 0xc42d3a45a0, 0xc42d377ca0, 0xc42d377ce0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xffffffff, 0x0, 0x19, 0xc4291097e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xc42d377ca0, 0xc42d377d00, 0xc429109878, 0x558012, 0x588a5743, 0x1eba63a9)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xed01c4e43, 0x0, 0x0, 0x5311)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xc42d377ca0, 0xc42d377d20, 0xa91b40, 0xc42d377d40, 0xc42d377ca0, 0xc42d377d40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xc420246100, 0xc42d3513d0, 0xc429109a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xc42d377ca0, 0xc42d377cc0, 0x4e, 0xc420314b18, 0x0, 0xc420314b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d3a4480, 0xc42d1eaf00, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ece00, 0xc420331380, 0x10265d8, 0xc42d3a4540, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ece00, 0xc42d3a4540)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ece00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 5564 [select, 13 minutes]:\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc4271a6840)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855\n\ngoroutine 8552 [chan send, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc443d482d0, 0xc42e9d88a0, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 8865 [chan receive, 10 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc42443d220, 0xc42c0510e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47\nnet/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc42443d220, 0xc42c0510e0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc42443d220, 0xc42c0510e0)\n\t/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc44a36e360, 0xc42c0510e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab\nnet/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc44a36e360, 0xc42c0510e0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc44a36e360, 0xc42c0510e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc44a36e340, 0xc42c050f00)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2\nnet/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc44a36e340, 0xc42c050f00)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc44a36e320, 0xc42c050f00)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c\nnet/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc44a36e320, 0xc42c050f00)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc43d0752b0, 0xc42c050f00)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449\nnet/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc43d0752b0, 0xc42c050e10)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc43d0752b0, 0xc42c050e10)\n\t/usr/local/go/src/net/http/server.go:2202 +0x7d\nnet/http.(*conn).serve(0xc4446eb300, 0x1038440, 0xc44628de40)\n\t/usr/local/go/src/net/http/server.go:1579 +0x4b7\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:2293 +0x44d\n\ngoroutine 9714 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4394e09f0, 0xc4394e0a20, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4394e09f0, 0xc43cf2a9c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4394e09f0, 0xc43c2a7ea0, 0x0, 0xc439ccf0c0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4394e09f0, 0xa74a80, 0xc43c2a7ea0, 0x7fad6035c8e0, 0xc4394e09f0, 0x1038500, 0xc4394e09c0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4394e09c0, 0xa74a80, 0xc43c2a7ea0, 0xc43cf2a8a0, 0xc43cf2a8c0, 0x7fad6035c8e0, 0xc4394e09c0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4394e09c0, 0xa74a80, 0xc43c2a7ea0, 0x7fad6035c8e0, 0xc4394e09c0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4394e09c0, 0xa74a80, 0xc43c2a7ea0, 0xc43cf2a8a0, 0xc43cf2a8e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xffffffff, 0x0, 0x19, 0xc449abf7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xc43cf2a8a0, 0xc43cf2a920, 0xc449abf878, 0x558012, 0x588a586d, 0x395d694c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xed01c4f6d, 0x0, 0x0, 0xa613d)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xc43cf2a8a0, 0xc43cf2a940, 0xa91b40, 0xc43cf2a960, 0xc43cf2a8a0, 0xc43cf2a960)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xc420246100, 0xc43c812a20, 0xc449abfa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xc43cf2a8a0, 0xc43cf2a8c0, 0x37fa, 0xc420046b18, 0x0, 0xc420046b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43dff1680, 0xc42b6f39e0, 0xc420179d60, 0x0, 0x0, 0x7fad625dc000, 0xc41eac8542)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87d00, 0xc420331380, 0x10265c0, 0xc43dff16b0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87d00, 0xc43dff16b0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87d00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10277 [chan receive, 8 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc44060ec80, 0xc4200d5860)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47\nnet/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc44060ec80, 0xc4200d5860)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc44060ec80, 0xc4200d5860)\n\t/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc43e90fea0, 0xc4200d5860)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab\nnet/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc43e90fea0, 0xc4200d5860)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc43e90fea0, 0xc4200d5860)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc43e90fe80, 0xc4200d5680)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2\nnet/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc43e90fe80, 0xc4200d5680)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc43e90fe60, 0xc4200d5680)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c\nnet/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc43e90fe60, 0xc4200d5680)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4201b40d0, 0xc4200d5680)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449\nnet/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4201b40d0, 0xc4200d5590)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4201b40d0, 0xc4200d5590)\n\t/usr/local/go/src/net/http/server.go:2202 +0x7d\nnet/http.(*conn).serve(0xc430fd1580, 0x1038440, 0xc439995f80)\n\t/usr/local/go/src/net/http/server.go:1579 +0x4b7\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:2293 +0x44d\n\ngoroutine 8984 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4440a2690, 0xc4440a26c0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4440a2690, 0xc43a68d420, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4440a2690, 0xc43c59e980, 0x0, 0xc441187ea8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4440a2690, 0xa74a80, 0xc43c59e980, 0x7fad6035c8e0, 0xc4440a2690, 0x1038500, 0xc4440a2660)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4440a2660, 0xa74a80, 0xc43c59e980, 0xc43a68d300, 0xc43a68d320, 0x7fad6035c8e0, 0xc4440a2660, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4440a2660, 0xa74a80, 0xc43c59e980, 0x7fad6035c8e0, 0xc4440a2660, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4440a2660, 0xa74a80, 0xc43c59e980, 0xc43a68d300, 0xc43a68d360, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xffffffff, 0x0, 0x19, 0xc43d6cf7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xc43a68d300, 0xc43a68d380, 0xc43d6cf878, 0x558012, 0x588a5810, 0xd1afb8b)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xed01c4f10, 0x0, 0x0, 0xa55a3)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xc43a68d300, 0xc43a68d3a0, 0xa91b40, 0xc43a68d3c0, 0xc43a68d300, 0xc43a68d3c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xc420246100, 0xc4418b35d0, 0xc43d6cfa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xc43a68d300, 0xc43a68d320, 0x7965, 0xc420e1db18, 0x0, 0xc420e1db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43caf49f0, 0xc4446d9da0, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0d00, 0xc420331380, 0x10265c0, 0xc43caf4b10, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0d00, 0xc43caf4b10)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0d00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8800 [chan receive, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc443047040, 0xc430b3fa40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47\nnet/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc443047040, 0xc430b3fa40)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc443047040, 0xc430b3fa40)\n\t/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc431d568a0, 0xc430b3fa40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab\nnet/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc431d568a0, 0xc430b3fa40)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc431d568a0, 0xc430b3fa40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc431d56880, 0xc430b3f860)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2\nnet/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc431d56880, 0xc430b3f860)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc431d56860, 0xc430b3f860)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c\nnet/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc431d56860, 0xc430b3f860)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4421cb5f0, 0xc430b3f860)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449\nnet/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4421cb5f0, 0xc430b3f770)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4421cb5f0, 0xc430b3f770)\n\t/usr/local/go/src/net/http/server.go:2202 +0x7d\nnet/http.(*conn).serve(0xc448e5be80, 0x1038440, 0xc421344280)\n\t/usr/local/go/src/net/http/server.go:1579 +0x4b7\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:2293 +0x44d\n\ngoroutine 8861 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43bceed20, 0xc43bceed50, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43bceed20, 0xc42aa56dc0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43bceed20, 0xc442132a60, 0x0, 0xc4398152c8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43bceed20, 0xa74a80, 0xc442132a60, 0x7fad6035c8e0, 0xc43bceed20, 0x1038500, 0xc43bceecf0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43bceecf0, 0xa74a80, 0xc442132a60, 0xc42aa56cc0, 0xc42aa56ce0, 0x7fad6035c8e0, 0xc43bceecf0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43bceecf0, 0xa74a80, 0xc442132a60, 0x7fad6035c8e0, 0xc43bceecf0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43bceecf0, 0xa74a80, 0xc442132a60, 0xc42aa56cc0, 0xc42aa56d00, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xffffffff, 0x0, 0x19, 0xc44a87d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xc42aa56cc0, 0xc42aa56d20, 0xc44a87d878, 0x558012, 0x588a57f5, 0x3260f940)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xed01c4ef5, 0x0, 0x0, 0x12c304)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xc42aa56cc0, 0xc42aa56d40, 0xa91b40, 0xc42aa56d60, 0xc42aa56cc0, 0xc42aa56d60)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xc420246100, 0xc42b6f1ff0, 0xc44a87da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xc42aa56cc0, 0xc42aa56ce0, 0x90b4, 0xc420315b18, 0x0, 0xc420315b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc448e5c270, 0xc42a5f5800, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f1a9759)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87600, 0xc420331380, 0x10265c0, 0xc448e5c2a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87600, 0xc448e5c2a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 12779 [semacquire]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42c690660, 0xc42c690690, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42c690660, 0xc43b978ca0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42c690660, 0xc44130b400, 0x0, 0xc42bcc9e48, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42c690660, 0xa74a80, 0xc44130b400, 0x7fad6035c8e0, 0xc42c690660, 0x1038500, 0xc42c690630)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c690630, 0xa74a80, 0xc44130b400, 0xc43b978b80, 0xc43b978ba0, 0x7fad6035c8e0, 0xc42c690630, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c690630, 0xa74a80, 0xc44130b400, 0x7fad6035c8e0, 0xc42c690630, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c690630, 0xa74a80, 0xc44130b400, 0xc43b978b80, 0xc43b978bc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0xffffffff, 0x0, 0x19, 0xc43132b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0xc43b978b80, 0xc43b978be0, 0xc43132b878, 0x558012, 0x588a5a4d, 0x3827c282)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0x41167e, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0xc43b978b80, 0xc43b978c00, 0xa91b40, 0xc43b978c40, 0xc43b978b80, 0xc43b978c40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0xc420246100, 0xc42d8d3990, 0xc43132ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0xc43b978b80, 0xc43b978ba0, 0x7629, 0xc420e1cb18, 0x0, 0xc420e1cb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d7d11d0, 0xc440cd3b60, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f32cb4d)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1300, 0xc420331380, 0x10265c0, 0xc42d7d1200, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1300, 0xc42d7d1200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8405 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc42b868fc0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42b868fc0, 0x159d0153bd2, 0x159d04d14b2, 0xc428411f40, 0x1, 0x1, 0xc42b868f00, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:272 +0x7e\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42b868fc0, 0xc42b868f00, 0x0, 0xc42ba1ac00, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42b868fc0, 0xa934c0, 0xc42b868f00, 0x7fad6035c8e0, 0xc42b868fc0, 0x1038500, 0xc42b868f90)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42b868f90, 0xa934c0, 0xc42b868f00, 0xc435021120, 0xc435021140, 0x7fad6035c8e0, 0xc42b868f90, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b868f90, 0xa934c0, 0xc42b868f00, 0x7fad6035c8e0, 0xc42b868f90, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42b868f90, 0xa934c0, 0xc42b868f00, 0xc435021120, 0xc4350211a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xffffffff, 0x0, 0x19, 0xc4454c77e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xc435021120, 0xc4350211c0, 0xc4454c7878, 0x558012, 0x588a57a4, 0x367107c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xed01c4ea4, 0x0, 0x0, 0x17d1d5)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xc435021120, 0xc4350211e0, 0xa91b40, 0xc435021220, 0xc435021120, 0xc435021220)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xc420246100, 0xc42ba1abc0, 0xc4454c7a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xc435021120, 0xc435021140, 0x41, 0xc420e1cb18, 0x0, 0xc420e1cb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42b868de0, 0xc4275212c0, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc426516900, 0xc420331380, 0x10265d8, 0xc42b868ea0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc426516900, 0xc42b868ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc43e3ca700, 0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc426516900)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8649 [chan receive, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc42443d680, 0xc430b3e4b0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47\nnet/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc42443d680, 0xc430b3e4b0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc42443d680, 0xc430b3e4b0)\n\t/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc43333f920, 0xc430b3e4b0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab\nnet/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc43333f920, 0xc430b3e4b0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc43333f920, 0xc430b3e4b0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc43333f8e0, 0xc430b3e2d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2\nnet/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc43333f8e0, 0xc430b3e2d0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc43333f8c0, 0xc430b3e2d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c\nnet/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc43333f8c0, 0xc430b3e2d0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc427d22f70, 0xc430b3e2d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449\nnet/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc427d22f70, 0xc430b3e1e0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc427d22f70, 0xc430b3e1e0)\n\t/usr/local/go/src/net/http/server.go:2202 +0x7d\nnet/http.(*conn).serve(0xc43cb60300, 0x1038440, 0xc427adb300)\n\t/usr/local/go/src/net/http/server.go:1579 +0x4b7\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:2293 +0x44d\n\ngoroutine 8568 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43d29c360, 0xc43d29c390, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43d29c360, 0xc442b641e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43d29c360, 0xc43dcc7da0, 0x0, 0xc42e11d168, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43d29c360, 0xa74a80, 0xc43dcc7da0, 0x7fad6035c8e0, 0xc43d29c360, 0x1038500, 0xc43d29c330)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43d29c330, 0xa74a80, 0xc43dcc7da0, 0xc442b640e0, 0xc442b64100, 0x7fad6035c8e0, 0xc43d29c330, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d29c330, 0xa74a80, 0xc43dcc7da0, 0x7fad6035c8e0, 0xc43d29c330, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43d29c330, 0xa74a80, 0xc43dcc7da0, 0xc442b640e0, 0xc442b64120, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xffffffff, 0x0, 0x19, 0xc4389e57e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xc442b640e0, 0xc442b64140, 0xc4389e5878, 0x558012, 0x588a59b8, 0x382e051)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xed01c50b8, 0x0, 0x0, 0x11251d)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xc442b640e0, 0xc442b64160, 0xa91b40, 0xc442b64180, 0xc442b640e0, 0xc442b64180)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xc420246100, 0xc42d55b730, 0xc4389e5a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xc442b640e0, 0xc442b64100, 0x4fe5, 0xc420e1db18, 0x0, 0xc420e1db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc440461890, 0xc43e024cc0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f700, 0xc420331380, 0x10265c0, 0xc4404618c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f700, 0xc4404618c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f700)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10660 [semacquire, 6 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc432995dd0, 0xc432995ef0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc432995dd0, 0xc4448c7820, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc432995dd0, 0xc43efe8fc0, 0x0, 0xc435bf2050, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc432995dd0, 0xa74a80, 0xc43efe8fc0, 0x7fad6035c8e0, 0xc432995dd0, 0x1038500, 0xc432995cb0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc432995cb0, 0xa74a80, 0xc43efe8fc0, 0xc4448c7700, 0xc4448c7720, 0x7fad6035c8e0, 0xc432995cb0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc432995cb0, 0xa74a80, 0xc43efe8fc0, 0x7fad6035c8e0, 0xc432995cb0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc432995cb0, 0xa74a80, 0xc43efe8fc0, 0xc4448c7700, 0xc4448c7740, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xffffffff, 0x0, 0x19, 0xc424dab7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xc4448c7700, 0xc4448c7760, 0xc424dab878, 0x558012, 0x588a591e, 0x1000faff)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xed01c501e, 0x0, 0x0, 0x177897)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xc4448c7700, 0xc4448c7780, 0xa91b40, 0xc4448c77a0, 0xc4448c7700, 0xc4448c77a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xc420246100, 0xc435c37ab0, 0xc424daba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xc4448c7700, 0xc4448c7720, 0x46f6, 0xc42023bb18, 0x0, 0xc42023bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc432dde4e0, 0xc428920ae0, 0xc420179d60, 0x0, 0x0, 0x50, 0x4)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5d00, 0xc420331380, 0x10265c0, 0xc432dde570, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5d00, 0xc432dde570)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5d00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8989 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43d26d620, 0xc43d26d650, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43d26d620, 0xc447e1ab20, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43d26d620, 0xc4481f7500, 0x0, 0xc43fc7dfc0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43d26d620, 0xa74a80, 0xc4481f7500, 0x7fad6035c8e0, 0xc43d26d620, 0x1038500, 0xc43d26d5f0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43d26d5f0, 0xa74a80, 0xc4481f7500, 0xc447e1a9e0, 0xc447e1aa20, 0x7fad6035c8e0, 0xc43d26d5f0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d26d5f0, 0xa74a80, 0xc4481f7500, 0x7fad6035c8e0, 0xc43d26d5f0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43d26d5f0, 0xa74a80, 0xc4481f7500, 0xc447e1a9e0, 0xc447e1aa40, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xffffffff, 0x0, 0x19, 0xc44a48d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xc447e1a9e0, 0xc447e1aa60, 0xc44a48d878, 0x558012, 0x588a5813, 0x36f6b00f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xed01c4f13, 0x0, 0x0, 0x126778)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xc447e1a9e0, 0xc447e1aa80, 0xa91b40, 0xc447e1aaa0, 0xc447e1a9e0, 0xc447e1aaa0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xc420246100, 0xc43fc7ce10, 0xc44a48da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xc447e1a9e0, 0xc447e1aa20, 0x2caa, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc441afb290, 0xc4431f7aa0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1c00, 0xc420331380, 0x10265c0, 0xc441afb2c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1c00, 0xc441afb2c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1c00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 11286 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc448a02fc0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc4351a0d10, 0xc448a02fc0, 0x10343c0, 0xc4201e3040)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326\n\ngoroutine 8325 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a56d7a0, 0xc42a56d7d0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a56d7a0, 0xc437cba8a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a56d7a0, 0xc437d3c020, 0x0, 0xc42a685c50, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a56d7a0, 0xa74a80, 0xc437d3c020, 0x7fad6035c8e0, 0xc42a56d7a0, 0x1038500, 0xc42a56d770)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a56d770, 0xa74a80, 0xc437d3c020, 0xc437cba7a0, 0xc437cba7c0, 0x7fad6035c8e0, 0xc42a56d770, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a56d770, 0xa74a80, 0xc437d3c020, 0x7fad6035c8e0, 0xc42a56d770, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a56d770, 0xa74a80, 0xc437d3c020, 0xc437cba7a0, 0xc437cba7e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xffffffff, 0x0, 0x19, 0xc4359997e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xc437cba7a0, 0xc437cba800, 0xc435999878, 0x558012, 0x588a5798, 0xdb76d51)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xc435999900, 0x0, 0x0, 0x3f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xc437cba7a0, 0xc437cba820, 0xa91b40, 0xc437cba840, 0xc437cba7a0, 0xc437cba840)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xc420246100, 0xc4284342d0, 0xc435999a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xc437cba7a0, 0xc437cba7c0, 0x9e4a, 0xc420e1ab18, 0x0, 0xc420e1ab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4248f1d70, 0xc448fe3680, 0xc420179d60, 0x0, 0x0, 0xc4202e17a8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3300, 0xc420331380, 0x10265c0, 0xc4248f1e00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3300, 0xc4248f1e00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8109 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc434146a50, 0xc434146a80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc434146a50, 0xc438f28420, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc434146a50, 0xc438f8d420, 0x0, 0xc4341417c8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc434146a50, 0xa74a80, 0xc438f8d420, 0x7fad6035c8e0, 0xc434146a50, 0x1038500, 0xc434146a20)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc434146a20, 0xa74a80, 0xc438f8d420, 0xc438f282e0, 0xc438f28300, 0x7fad6035c8e0, 0xc434146a20, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc434146a20, 0xa74a80, 0xc438f8d420, 0x7fad6035c8e0, 0xc434146a20, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc434146a20, 0xa74a80, 0xc438f8d420, 0xc438f282e0, 0xc438f28340, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xffffffff, 0x0, 0x19, 0xc4215df7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xc438f282e0, 0xc438f28360, 0xc4215df878, 0x558012, 0x588a577a, 0xac8b930)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xed01c4e7a, 0x0, 0x0, 0x72310)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xc438f282e0, 0xc438f28380, 0xa91b40, 0xc438f283a0, 0xc438f282e0, 0xc438f283a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xc420246100, 0xc42276fc80, 0xc4215dfa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xc438f282e0, 0xc438f28300, 0x3e88, 0xc449a0ab18, 0x0, 0xc449a0ab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44151e120, 0xc423b1ec00, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa600, 0xc420331380, 0x10265c0, 0xc42516b200, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa600, 0xc42516b200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8323 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42c71a660, 0xc42c71a690, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42c71a660, 0xc437c84460, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42c71a660, 0xc437ce8f60, 0x0, 0xc42bc95ee0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42c71a660, 0xa74a80, 0xc437ce8f60, 0x7fad6035c8e0, 0xc42c71a660, 0x1038500, 0xc42c71a630)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c71a630, 0xa74a80, 0xc437ce8f60, 0xc437c84300, 0xc437c84320, 0x7fad6035c8e0, 0xc42c71a630, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c71a630, 0xa74a80, 0xc437ce8f60, 0x7fad6035c8e0, 0xc42c71a630, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c71a630, 0xa74a80, 0xc437ce8f60, 0xc437c84300, 0xc437c84340, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xffffffff, 0x0, 0x19, 0xc43b4ef7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xc437c84300, 0xc437c843a0, 0xc43b4ef878, 0x558012, 0x588a5798, 0xdbae98f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xed01c4e98, 0x0, 0x0, 0x2aa0b6)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xc437c84300, 0xc437c843c0, 0xa91b40, 0xc437c843e0, 0xc437c84300, 0xc437c843e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xc420246100, 0xc42a686a50, 0xc43b4efa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xc437c84300, 0xc437c84320, 0x859e, 0xc420049b18, 0x0, 0xc420049b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42f4079b0, 0xc4296fa6c0, 0xc420179d60, 0x0, 0x0, 0x646e696b040a100a, 0x61646174656d0812)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4277c2f00, 0xc420331380, 0x10265c0, 0xc42f4079e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4277c2f00, 0xc42f4079e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4277c2f00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8985 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43e0c4e70, 0xc43e0c4ea0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43e0c4e70, 0xc43d8820e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43e0c4e70, 0xc4496c16a0, 0x0, 0xc4425b13c0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43e0c4e70, 0xa74a80, 0xc4496c16a0, 0x7fad6035c8e0, 0xc43e0c4e70, 0x1038500, 0xc43e0c4e40)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43e0c4e40, 0xa74a80, 0xc4496c16a0, 0xc443da9fe0, 0xc43d882000, 0x7fad6035c8e0, 0xc43e0c4e40, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c4e40, 0xa74a80, 0xc4496c16a0, 0x7fad6035c8e0, 0xc43e0c4e40, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43e0c4e40, 0xa74a80, 0xc4496c16a0, 0xc443da9fe0, 0xc43d882020, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xffffffff, 0x0, 0x19, 0xc44a66f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xc443da9fe0, 0xc43d882040, 0xc44a66f878, 0x558012, 0x588a5813, 0x328ff5c4)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xed01c4f13, 0x0, 0x0, 0x4665b)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xc443da9fe0, 0xc43d882060, 0xa91b40, 0xc43d882080, 0xc443da9fe0, 0xc43d882080)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xc420246100, 0xc4425b0920, 0xc44a66fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xc443da9fe0, 0xc43d882000, 0x1ba7, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43e0c46c0, 0xc443c35800, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a1400, 0xc420331380, 0x10265c0, 0xc43e0c46f0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a1400, 0xc43e0c46f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a1400)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 5262 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc420497484)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc420497480)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x24560ccb27db9cc2)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc44240f1a0, 0xc422334c2d, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44240e3c0, 0xc44240f1a0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44240e3c0, 0xc42992e4e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44240e3c0, 0xc429a05260, 0x0, 0xc442410570, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44240e3c0, 0xa74a80, 0xc429a05260, 0x7fad6035c8e0, 0xc44240e3c0, 0x1038500, 0xc44240e390)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44240e390, 0xa74a80, 0xc429a05260, 0xc429961040, 0xc4299610a0, 0x7fad6035c8e0, 0xc44240e390, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44240e390, 0xa74a80, 0xc429a05260, 0x7fad6035c8e0, 0xc44240e390, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44240e390, 0xa74a80, 0xc429a05260, 0xc429961040, 0xc4299610c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xffffffff, 0x0, 0x19, 0xc42910d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xc429961040, 0xc4299610e0, 0xc42910d878, 0x558012, 0x588a575f, 0x3a21669b)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xed01c4e5f, 0x0, 0x0, 0x185c22)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xc429961040, 0xc429961120, 0xa91b40, 0xc429961140, 0xc429961040, 0xc429961140)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xc420246100, 0xc4435da830, 0xc42910da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xc429961040, 0xc4299610a0, 0xbace, 0xc449cdab18, 0x0, 0xc449cdab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43dff0c60, 0xc43bba5e60, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4eb00, 0xc420331380, 0x10265c0, 0xc43dff0c90, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4eb00, 0xc43dff0c90)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4eb00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8147 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4439c32f0, 0xc4439c3320, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4439c32f0, 0xc438b0e400, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4439c32f0, 0xc438b63b00, 0x0, 0xc448466cb0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4439c32f0, 0xa74a80, 0xc438b63b00, 0x7fad6035c8e0, 0xc4439c32f0, 0x1038500, 0xc4439c32c0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4439c32c0, 0xa74a80, 0xc438b63b00, 0xc438b0e2a0, 0xc438b0e2c0, 0x7fad6035c8e0, 0xc4439c32c0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4439c32c0, 0xa74a80, 0xc438b63b00, 0x7fad6035c8e0, 0xc4439c32c0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4439c32c0, 0xa74a80, 0xc438b63b00, 0xc438b0e2a0, 0xc438b0e2e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xffffffff, 0x0, 0x19, 0xc43d1517e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xc438b0e2a0, 0xc438b0e300, 0xc43d151878, 0x558012, 0x588a577d, 0x3873ef2f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xed01c4e7d, 0x0, 0x0, 0xe3b6e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xc438b0e2a0, 0xc438b0e320, 0xa91b40, 0xc438b0e360, 0xc438b0e2a0, 0xc438b0e360)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xc420246100, 0xc44757c9c0, 0xc43d151a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xc438b0e2a0, 0xc438b0e2c0, 0xa28a, 0xc420319b18, 0x0, 0xc420319b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442346660, 0xc43d52d740, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abf00, 0xc420331380, 0x10265c0, 0xc442346690, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abf00, 0xc442346690)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abf00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10458 [chan send, 6 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc42c051590, 0xc438e702a0, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 11262 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d097860, 0xc42d097890, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d097860, 0xc443553c80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d097860, 0xc446b26080, 0x0, 0xc4333feaf8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d097860, 0xa74a80, 0xc446b26080, 0x7fad6035c8e0, 0xc42d097860, 0x1038500, 0xc42d097830)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d097830, 0xa74a80, 0xc446b26080, 0xc443553b80, 0xc443553ba0, 0x7fad6035c8e0, 0xc42d097830, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d097830, 0xa74a80, 0xc446b26080, 0x7fad6035c8e0, 0xc42d097830, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d097830, 0xa74a80, 0xc446b26080, 0xc443553b80, 0xc443553bc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0xffffffff, 0x0, 0x19, 0xc4491af7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0xc443553b80, 0xc443553be0, 0xc4491af878, 0x558012, 0x588a5978, 0x11c865da)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0x41167e, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0xc443553b80, 0xc443553c00, 0xa91b40, 0xc443553c20, 0xc443553b80, 0xc443553c20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0xc420246100, 0xc4335ea6b0, 0xc4491afa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0xc443553b80, 0xc443553ba0, 0x75ed, 0xc440d64b18, 0x0, 0xc440d64b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42b3ad230, 0xc44415d860, 0xc420179d60, 0x0, 0x0, 0x10, 0xc439ad6e58)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fb300, 0xc420331380, 0x10265c0, 0xc42b3ad2f0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fb300, 0xc42b3ad2f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fb300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8113 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc440d230e0, 0xc440d23110, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc440d230e0, 0xc438dafe60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc440d230e0, 0xc438de3a60, 0x0, 0xc440d24190, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc440d230e0, 0xa74a80, 0xc438de3a60, 0x7fad6035c8e0, 0xc440d230e0, 0x1038500, 0xc440d230b0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc440d230b0, 0xa74a80, 0xc438de3a60, 0xc438dafd20, 0xc438dafd40, 0x7fad6035c8e0, 0xc440d230b0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440d230b0, 0xa74a80, 0xc438de3a60, 0x7fad6035c8e0, 0xc440d230b0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc440d230b0, 0xa74a80, 0xc438de3a60, 0xc438dafd20, 0xc438dafd60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xffffffff, 0x0, 0x19, 0xc42d70f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xc438dafd20, 0xc438dafda0, 0xc42d70f878, 0x558012, 0x588a577a, 0xcf101a6)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xed01c4e7a, 0x0, 0x0, 0x692eb)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xc438dafd20, 0xc438dafdc0, 0xa91b40, 0xc438dafde0, 0xc438dafd20, 0xc438dafde0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xc420246100, 0xc446b14800, 0xc42d70fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xc438dafd20, 0xc438dafd40, 0x3746, 0xc420044b18, 0x0, 0xc420044b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc446b13a70, 0xc446b0c120, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4267ab300, 0xc420331380, 0x10265c0, 0xc446b13b30, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4267ab300, 0xc446b13b30)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4267ab300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8073 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc423f8c600, 0xc423f8c630, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc423f8c600, 0xc433e06380, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc423f8c600, 0xc433f2bbe0, 0x0, 0xc423f8e140, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc423f8c600, 0xa74a80, 0xc433f2bbe0, 0x7fad6035c8e0, 0xc423f8c600, 0x1038500, 0xc423f8c5d0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc423f8c5d0, 0xa74a80, 0xc433f2bbe0, 0xc433e06180, 0xc433e061e0, 0x7fad6035c8e0, 0xc423f8c5d0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423f8c5d0, 0xa74a80, 0xc433f2bbe0, 0x7fad6035c8e0, 0xc423f8c5d0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc423f8c5d0, 0xa74a80, 0xc433f2bbe0, 0xc433e06180, 0xc433e06200, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xffffffff, 0x0, 0x19, 0xc448f057e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xc433e06180, 0xc433e06220, 0xc448f05878, 0x558012, 0x588a57b6, 0xd019425)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xed01c4eb6, 0x0, 0x0, 0xc1691)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xc433e06180, 0xc433e062a0, 0xa91b40, 0xc433e062c0, 0xc433e06180, 0xc433e062c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xc420246100, 0xc424ba9bc0, 0xc448f05a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xc433e06180, 0xc433e061e0, 0x66ce, 0xc420319b18, 0x0, 0xc420319b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42378aae0, 0xc42779ec60, 0xc420179d60, 0x0, 0x0, 0x4d1279726f746973, 0x6f642e7865646e69)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4228bc600, 0xc420331380, 0x10265c0, 0xc42378ab10, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4228bc600, 0xc42378ab10)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4228bc600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8988 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441afb560, 0xc441afb590, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441afb560, 0xc44693fc00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441afb560, 0xc44693f700, 0x0, 0xc448040940, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441afb560, 0xa74a80, 0xc44693f700, 0x7fad6035c8e0, 0xc441afb560, 0x1038500, 0xc441afb530)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc441afb530, 0xa74a80, 0xc44693f700, 0xc44693fae0, 0xc44693fb00, 0x7fad6035c8e0, 0xc441afb530, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb530, 0xa74a80, 0xc44693f700, 0x7fad6035c8e0, 0xc441afb530, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc441afb530, 0xa74a80, 0xc44693f700, 0xc44693fae0, 0xc44693fb20, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xffffffff, 0x0, 0x19, 0xc44a48f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xc44693fae0, 0xc44693fb40, 0xc44a48f878, 0x558012, 0x588a5813, 0x36e5e965)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xed01c4f13, 0x0, 0x0, 0x11cf1)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xc44693fae0, 0xc44693fb60, 0xa91b40, 0xc44693fba0, 0xc44693fae0, 0xc44693fba0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xc420246100, 0xc448040050, 0xc44a48fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xc44693fae0, 0xc44693fb00, 0x272, 0xc420e1db18, 0x0, 0xc420e1db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc441afb0b0, 0xc443c35e00, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1b00, 0xc420331380, 0x10265c0, 0xc441afb320, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1b00, 0xc441afb320)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1b00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9883 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437c77920, 0xc437c779b0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437c77920, 0xc44104b6c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437c77920, 0xc445c70b00, 0x0, 0xc438b972b0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437c77920, 0xa74a80, 0xc445c70b00, 0x7fad6035c8e0, 0xc437c77920, 0x1038500, 0xc437c77890)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc437c77890, 0xa74a80, 0xc445c70b00, 0xc44104b5c0, 0xc44104b5e0, 0x7fad6035c8e0, 0xc437c77890, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c77890, 0xa74a80, 0xc445c70b00, 0x7fad6035c8e0, 0xc437c77890, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc437c77890, 0xa74a80, 0xc445c70b00, 0xc44104b5c0, 0xc44104b600, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xffffffff, 0x0, 0x19, 0xc44145b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xc44104b5c0, 0xc44104b620, 0xc44145b878, 0x558012, 0x588a588b, 0x32d2020d)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xed01c4f8b, 0x0, 0x0, 0x8457f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xc44104b5c0, 0xc44104b640, 0xa91b40, 0xc44104b660, 0xc44104b5c0, 0xc44104b660)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xc420246100, 0xc438c390c0, 0xc44145ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xc44104b5c0, 0xc44104b5e0, 0x2edd, 0xc42023bb18, 0x0, 0xc42023bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc437cc1800, 0xc4411971a0, 0xc420179d60, 0x0, 0x0, 0xc439252d40, 0x646e696b)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b5c00, 0xc420331380, 0x10265c0, 0xc437cc1830, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b5c00, 0xc437cc1830)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b5c00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8518 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42210b020, 0xc42210b050, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42210b020, 0xc433b7ab00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42210b020, 0xc433bba280, 0x0, 0xc42b6f0760, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42210b020, 0xa74a80, 0xc433bba280, 0x7fad6035c8e0, 0xc42210b020, 0x1038500, 0xc42210aff0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42210aff0, 0xa74a80, 0xc433bba280, 0xc433b7a9a0, 0xc433b7a9c0, 0x7fad6035c8e0, 0xc42210aff0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210aff0, 0xa74a80, 0xc433bba280, 0x7fad6035c8e0, 0xc42210aff0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42210aff0, 0xa74a80, 0xc433bba280, 0xc433b7a9a0, 0xc433b7aa20, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xffffffff, 0x0, 0x19, 0xc4425a77e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xc433b7a9a0, 0xc433b7aa40, 0xc4425a7878, 0x558012, 0x588a57b9, 0x3b697901)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xed01c4eb9, 0x0, 0x0, 0x2ce5d)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xc433b7a9a0, 0xc433b7aa60, 0xa91b40, 0xc433b7aa80, 0xc433b7a9a0, 0xc433b7aa80)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xc420246100, 0xc4223f7d60, 0xc4425a7a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xc433b7a9a0, 0xc433b7a9c0, 0x1aba, 0xc420318b18, 0x0, 0xc420318b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42210a8d0, 0xc428757f20, 0xc420179d60, 0x0, 0x0, 0x4d1279726f746973, 0x6f642e7865646e69)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f100, 0xc420331380, 0x10265c0, 0xc42210a990, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f100, 0xc42210a990)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f100)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8531 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4269fc3c0, 0xc4269fc3f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4269fc3c0, 0xc434061f00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4269fc3c0, 0xc434200bc0, 0x0, 0xc426a916c0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4269fc3c0, 0xa74a80, 0xc434200bc0, 0x7fad6035c8e0, 0xc4269fc3c0, 0x1038500, 0xc4269fc390)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4269fc390, 0xa74a80, 0xc434200bc0, 0xc434061da0, 0xc434061de0, 0x7fad6035c8e0, 0xc4269fc390, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4269fc390, 0xa74a80, 0xc434200bc0, 0x7fad6035c8e0, 0xc4269fc390, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4269fc390, 0xa74a80, 0xc434200bc0, 0xc434061da0, 0xc434061e00, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xffffffff, 0x0, 0x19, 0xc43fccd7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xc434061da0, 0xc434061e20, 0xc43fccd878, 0x558012, 0x588a57b6, 0xbef1596)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xed01c4eb6, 0x0, 0x0, 0x121c53)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xc434061da0, 0xc434061e60, 0xa91b40, 0xc434061e80, 0xc434061da0, 0xc434061e80)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xc420246100, 0xc4284b1a60, 0xc43fccda70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xc434061da0, 0xc434061de0, 0xaf25, 0xc420e18b18, 0x0, 0xc420e18b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc428928c60, 0xc428756720, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5d1194)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422e08500, 0xc420331380, 0x10265c0, 0xc428928c90, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422e08500, 0xc428928c90)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422e08500)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10239 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43627f2c0, 0xc43627f350, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43627f2c0, 0xc43b608220, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43627f2c0, 0xc43b257320, 0x0, 0xc43772da00, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43627f2c0, 0xa74a80, 0xc43b257320, 0x7fad6035c8e0, 0xc43627f2c0, 0x1038500, 0xc43627f230)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43627f230, 0xa74a80, 0xc43b257320, 0xc43b608120, 0xc43b608140, 0x7fad6035c8e0, 0xc43627f230, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43627f230, 0xa74a80, 0xc43b257320, 0x7fad6035c8e0, 0xc43627f230, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43627f230, 0xa74a80, 0xc43b257320, 0xc43b608120, 0xc43b608160, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xffffffff, 0x0, 0x19, 0xc43b0937e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xc43b608120, 0xc43b608180, 0xc43b093878, 0x558012, 0x588a58c4, 0x10f2f5bc)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xed01c4fc4, 0x0, 0x0, 0x13506b)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xc43b608120, 0xc43b6081a0, 0xa91b40, 0xc43b6081c0, 0xc43b608120, 0xc43b6081c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xc420246100, 0xc4377ba030, 0xc43b093a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xc43b608120, 0xc43b608140, 0x62d8, 0xc420239b18, 0x0, 0xc420239b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4362ce720, 0xc433d43020, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438723b00, 0xc420331380, 0x10265c0, 0xc4362ced50, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438723b00, 0xc4362ced50)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438723b00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 7441 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44175c660, 0xc44175c690, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44175c660, 0xc447b4e520, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44175c660, 0xc44177a020, 0x0, 0xc432bf7c80, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44175c660, 0xa74a80, 0xc44177a020, 0x7fad6035c8e0, 0xc44175c660, 0x1038500, 0xc44175c630)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44175c630, 0xa74a80, 0xc44177a020, 0xc447b4e3e0, 0xc447b4e400, 0x7fad6035c8e0, 0xc44175c630, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44175c630, 0xa74a80, 0xc44177a020, 0x7fad6035c8e0, 0xc44175c630, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44175c630, 0xa74a80, 0xc44177a020, 0xc447b4e3e0, 0xc447b4e420, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xffffffff, 0x0, 0x19, 0xc43808b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xc447b4e3e0, 0xc447b4e440, 0xc43808b878, 0x558012, 0x588a59d2, 0x1224f8ee)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xed01c50d2, 0x0, 0x0, 0x1827c8)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xc447b4e3e0, 0xc447b4e460, 0xa91b40, 0xc447b4e480, 0xc447b4e3e0, 0xc447b4e480)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xc420246100, 0xc4351c4880, 0xc43808ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xc447b4e3e0, 0xc447b4e400, 0x7c56, 0xc42023cb18, 0x0, 0xc42023cb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4424521b0, 0xc44103c180, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ed5c013)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6000, 0xc420331380, 0x10265c0, 0xc4424524e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6000, 0xc4424524e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 5563 [IO wait, 4 minutes]:\nnet.runtime_pollWait(0x7fad6039cca0, 0x72, 0xc)\n\t/usr/local/go/src/runtime/netpoll.go:160 +0x59\nnet.(*pollDesc).wait(0xc43d5271e0, 0x72, 0xc4203178d8, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38\nnet.(*pollDesc).waitRead(0xc43d5271e0, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34\nnet.(*netFD).Read(0xc43d527180, 0xc43e0d6000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_unix.go:243 +0x1a1\nnet.(*conn).Read(0xc43a4a07d8, 0xc43e0d6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)\n\t/usr/local/go/src/net/net.go:173 +0x70\nbufio.(*Reader).fill(0xc43e447a40)\n\t/usr/local/go/src/bufio/bufio.go:97 +0x10c\nbufio.(*Reader).Read(0xc43e447a40, 0xc4271a67b8, 0x9, 0x9, 0x43fe80, 0xc431af8170, 0x0)\n\t/usr/local/go/src/bufio/bufio.go:209 +0x1bc\nio.ReadAtLeast(0x102f380, 0xc43e447a40, 0xc4271a67b8, 0x9, 0x9, 0x9, 0xc420317ae8, 0x451a90, 0xc43ba02dc0)\n\t/usr/local/go/src/io/io.go:307 +0xa4\nio.ReadFull(0x102f380, 0xc43e447a40, 0xc4271a67b8, 0x9, 0x9, 0xc431af8170, 0x0, 0x1)\n\t/usr/local/go/src/io/io.go:325 +0x58\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc4271a67b8, 0x9, 0x9, 0x102f380, 0xc43e447a40, 0x0, 0xc400000000, 0xc420317c80, 0x7f3105)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4271a6780, 0xf, 0xf, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc43e0c5410, 0xc43b9713e0, 0xc43b9713e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4271a6840, 0xc43e0c54a0, 0xb475b8)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc4271a6840)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc43a4a07d8, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc43a4a07d8)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac\n\ngoroutine 12131 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc443068600)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc42e0006f0, 0xc443068600, 0x10343c0, 0xc4201e3040)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326\n\ngoroutine 8986 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441afa270, 0xc441afa2a0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441afa270, 0xc44693ec00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441afa270, 0xc43d882460, 0x0, 0xc449c55ea0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441afa270, 0xa74a80, 0xc43d882460, 0x7fad6035c8e0, 0xc441afa270, 0x1038500, 0xc441afa240)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc441afa240, 0xa74a80, 0xc43d882460, 0xc44693eae0, 0xc44693eb00, 0x7fad6035c8e0, 0xc441afa240, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afa240, 0xa74a80, 0xc43d882460, 0x7fad6035c8e0, 0xc441afa240, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc441afa240, 0xa74a80, 0xc43d882460, 0xc44693eae0, 0xc44693eb20, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xffffffff, 0x0, 0x19, 0xc44a6717e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xc44693eae0, 0xc44693eb40, 0xc44a671878, 0x558012, 0x588a5813, 0x331cd64c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xed01c4f13, 0x0, 0x0, 0x67995)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xc44693eae0, 0xc44693eb60, 0xa91b40, 0xc44693eb80, 0xc44693eae0, 0xc44693eb80)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xc420246100, 0xc4425b1470, 0xc44a671a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xc44693eae0, 0xc44693eb00, 0x29ae, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43e0c5530, 0xc443c35b60, 0xc420179d60, 0x0, 0x0, 0xc4200e5800, 0x450280)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1700, 0xc420331380, 0x10265c0, 0xc43e0c5560, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1700, 0xc43e0c5560)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1700)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 5346 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204895c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204895c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x8ec308e871713ce)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc444ae6f30, 0xc420162418, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc444ae65d0, 0xc444ae6f30, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc444ae65d0, 0xc42b4b1660, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc444ae65d0, 0xc42b53f240, 0x0, 0xc444aea400, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc444ae65d0, 0xa74a80, 0xc42b53f240, 0x7fad6035c8e0, 0xc444ae65d0, 0x1038500, 0xc444ae65a0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc444ae65a0, 0xa74a80, 0xc42b53f240, 0xc42b4b0b60, 0xc42b4b0ba0, 0x7fad6035c8e0, 0xc444ae65a0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444ae65a0, 0xa74a80, 0xc42b53f240, 0x7fad6035c8e0, 0xc444ae65a0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc444ae65a0, 0xa74a80, 0xc42b53f240, 0xc42b4b0b60, 0xc42b4b0bc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xffffffff, 0x0, 0x19, 0xc4215bd7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xc42b4b0b60, 0xc42b4b0be0, 0xc4215bd878, 0x558012, 0x588a575f, 0x36ab2c69)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xed01c4e5f, 0x0, 0x0, 0xf2bf3)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xc42b4b0b60, 0xc42b4b0c20, 0xa91b40, 0xc42b4b0c40, 0xc42b4b0b60, 0xc42b4b0c40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xc420246100, 0xc444f1f710, 0xc4215bda70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xc42b4b0b60, 0xc42b4b0ba0, 0x8a5c, 0xc420313b18, 0x0, 0xc420313b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc444dba7e0, 0xc448c5cc60, 0xc420179d60, 0x0, 0x0, 0xc42ad71fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbef00, 0xc420331380, 0x10265c0, 0xc444dba810, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbef00, 0xc444dba810)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbef00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8982 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43b0b6780, 0xc43b0b67b0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43b0b6780, 0xc43f6e7f40, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43b0b6780, 0xc43aef1fc0, 0x0, 0xc442d71580, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43b0b6780, 0xa74a80, 0xc43aef1fc0, 0x7fad6035c8e0, 0xc43b0b6780, 0x1038500, 0xc43b0b6750)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43b0b6750, 0xa74a80, 0xc43aef1fc0, 0xc43f6e7e40, 0xc43f6e7e60, 0x7fad6035c8e0, 0xc43b0b6750, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b0b6750, 0xa74a80, 0xc43aef1fc0, 0x7fad6035c8e0, 0xc43b0b6750, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43b0b6750, 0xa74a80, 0xc43aef1fc0, 0xc43f6e7e40, 0xc43f6e7e80, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xffffffff, 0x0, 0x19, 0xc43d6d37e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xc43f6e7e40, 0xc43f6e7ea0, 0xc43d6d3878, 0x558012, 0x588a5810, 0xd2c314c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xed01c4f10, 0x0, 0x0, 0x2db06e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xc43f6e7e40, 0xc43f6e7ec0, 0xa91b40, 0xc43f6e7ee0, 0xc43f6e7e40, 0xc43f6e7ee0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xc420246100, 0xc441e55320, 0xc43d6d3a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xc43f6e7e40, 0xc43f6e7e60, 0x75b3, 0xc420e18b18, 0x0, 0xc420e18b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43caf41b0, 0xc4431f7320, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f3c36c4)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0800, 0xc420331380, 0x10265c0, 0xc43caf4450, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0800, 0xc43caf4450)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0800)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8569 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43fab54d0, 0xc43fab5500, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43fab54d0, 0xc43dad9d60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43fab54d0, 0xc447b4f340, 0x0, 0xc431af9238, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43fab54d0, 0xa74a80, 0xc447b4f340, 0x7fad6035c8e0, 0xc43fab54d0, 0x1038500, 0xc43fab54a0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43fab54a0, 0xa74a80, 0xc447b4f340, 0xc43dad9c60, 0xc43dad9c80, 0x7fad6035c8e0, 0xc43fab54a0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43fab54a0, 0xa74a80, 0xc447b4f340, 0x7fad6035c8e0, 0xc43fab54a0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43fab54a0, 0xa74a80, 0xc447b4f340, 0xc43dad9c60, 0xc43dad9ca0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xffffffff, 0x0, 0x19, 0xc4383637e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xc43dad9c60, 0xc43dad9cc0, 0xc438363878, 0x558012, 0x588a59d2, 0x123bfb30)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xed01c50d2, 0x0, 0x0, 0x29b015)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xc43dad9c60, 0xc43dad9ce0, 0xa91b40, 0xc43dad9d00, 0xc43dad9c60, 0xc43dad9d00)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xc420246100, 0xc4329c4260, 0xc438363a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xc43dad9c60, 0xc43dad9c80, 0x799d, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4432da030, 0xc44103c1e0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f228f32)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439775500, 0xc420331380, 0x10265c0, 0xc4432da060, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439775500, 0xc4432da060)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439775500)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 5378 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc420489c04)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc420489c00)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x46e19a0f49d894d)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc4416303f0, 0xc422334c2d, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441630330, 0xc4416303f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441630330, 0xc42935f6e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441630330, 0xc429429c60, 0x0, 0xc4418654e8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441630330, 0xa74a80, 0xc429429c60, 0x7fad6035c8e0, 0xc441630330, 0x1038500, 0xc441630300)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc441630300, 0xa74a80, 0xc429429c60, 0xc42935f0e0, 0xc42935f140, 0x7fad6035c8e0, 0xc441630300, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441630300, 0xa74a80, 0xc429429c60, 0x7fad6035c8e0, 0xc441630300, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc441630300, 0xa74a80, 0xc429429c60, 0xc42935f0e0, 0xc42935f180, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xffffffff, 0x0, 0x19, 0xc43ee3f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xc42935f0e0, 0xc42935f2c0, 0xc43ee3f878, 0x558012, 0x588a5760, 0x215ec54)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xed01c4e60, 0x0, 0x0, 0x100641)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xc42935f0e0, 0xc42935f300, 0xa91b40, 0xc42935f380, 0xc42935f0e0, 0xc42935f380)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xc420246100, 0xc4418b3970, 0xc43ee3fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xc42935f0e0, 0xc42935f140, 0xabbf, 0xc446d5eb18, 0x0, 0xc446d5eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4418540f0, 0xc440870fc0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f600, 0xc420331380, 0x10265c0, 0xc441854120, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f600, 0xc441854120)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10272 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc435b86420, 0xc435b871d0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc435b86420, 0xc4428c9da0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc435b86420, 0xc445d3f340, 0x0, 0xc43732bcb0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc435b86420, 0xa74a80, 0xc445d3f340, 0x7fad6035c8e0, 0xc435b86420, 0x1038500, 0xc435b863f0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc435b863f0, 0xa74a80, 0xc445d3f340, 0xc4428c9ca0, 0xc4428c9cc0, 0x7fad6035c8e0, 0xc435b863f0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b863f0, 0xa74a80, 0xc445d3f340, 0x7fad6035c8e0, 0xc435b863f0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc435b863f0, 0xa74a80, 0xc445d3f340, 0xc4428c9ca0, 0xc4428c9ce0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xffffffff, 0x0, 0x19, 0xc440b4b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xc4428c9ca0, 0xc4428c9d00, 0xc440b4b878, 0x558012, 0x588a58c7, 0x3389e516)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xed01c4fc7, 0x0, 0x0, 0x275370)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xc4428c9ca0, 0xc4428c9d20, 0xa91b40, 0xc4428c9d40, 0xc4428c9ca0, 0xc4428c9d40)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xc420246100, 0xc43748c240, 0xc440b4ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xc4428c9ca0, 0xc4428c9cc0, 0xabd9, 0xc420319b18, 0x0, 0xc420319b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc435bdfda0, 0xc433c3ec00, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ee82460)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc000, 0xc420331380, 0x10265c0, 0xc435bdfdd0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc000, 0xc435bdfdd0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8326 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4223bc990, 0xc4223bc9c0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4223bc990, 0xc437b94840, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4223bc990, 0xc437bd6340, 0x0, 0xc4223bb490, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4223bc990, 0xa74a80, 0xc437bd6340, 0x7fad6035c8e0, 0xc4223bc990, 0x1038500, 0xc4223bc960)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4223bc960, 0xa74a80, 0xc437bd6340, 0xc437b94700, 0xc437b94720, 0x7fad6035c8e0, 0xc4223bc960, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4223bc960, 0xa74a80, 0xc437bd6340, 0x7fad6035c8e0, 0xc4223bc960, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4223bc960, 0xa74a80, 0xc437bd6340, 0xc437b94700, 0xc437b94740, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xffffffff, 0x0, 0x19, 0xc4454c57e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xc437b94700, 0xc437b94780, 0xc4454c5878, 0x558012, 0x588a579b, 0x32a304c1)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xed01c4e9b, 0x0, 0x0, 0x1871af)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xc437b94700, 0xc437b947a0, 0xa91b40, 0xc437b947c0, 0xc437b94700, 0xc437b947c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xc420246100, 0xc42df97ad0, 0xc4454c5a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xc437b94700, 0xc437b94720, 0x8bd0, 0xc420e1db18, 0x0, 0xc420e1db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42deb6510, 0xc4296fb740, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f40378e)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516100, 0xc420331380, 0x10265c0, 0xc42deb6540, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516100, 0xc42deb6540)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516100)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8151 [chan receive, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc42d97bf40, 0xc42c0502d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47\nnet/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc42d97bf40, 0xc42c0502d0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc42d97bf40, 0xc42c0502d0)\n\t/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc43a5a1dc0, 0xc42c0502d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab\nnet/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc43a5a1dc0, 0xc42c0502d0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc43a5a1dc0, 0xc42c0502d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc43a5a1da0, 0xc42c0500f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2\nnet/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc43a5a1da0, 0xc42c0500f0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc43a5a1d80, 0xc42c0500f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c\nnet/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc43a5a1d80, 0xc42c0500f0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4292de340, 0xc42c0500f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449\nnet/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4292de340, 0xc42c050000)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4292de340, 0xc42c050000)\n\t/usr/local/go/src/net/http/server.go:2202 +0x7d\nnet/http.(*conn).serve(0xc43e55b580, 0x1038440, 0xc42b140bc0)\n\t/usr/local/go/src/net/http/server.go:1579 +0x4b7\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:2293 +0x44d\n\ngoroutine 8930 [chan send, 10 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc42c0510e0, 0xc445cf80c0, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 8391 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc42c212900)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc42c133760, 0xc42c212900, 0x10343c0, 0xc4201e3040)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326\n\ngoroutine 8242 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc444b24d4c)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*WaitGroup).Wait(0xc444b24d40)\n\t/usr/local/go/src/sync/waitgroup.go:131 +0x97\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc444b24d40, 0xc446b0daa0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e\n\ngoroutine 8747 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc443c341e0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc443ff5940, 0xc443c341e0, 0x10343c0, 0xc4201e3040)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326\n\ngoroutine 10276 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43cbb6690, 0xc43cbb66c0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43cbb6690, 0xc43e90f300, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43cbb6690, 0xc443e91ce0, 0x0, 0xc436eb9ff8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43cbb6690, 0xa74a80, 0xc443e91ce0, 0x7fad6035c8e0, 0xc43cbb6690, 0x1038500, 0xc43cbb6660)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43cbb6660, 0xa74a80, 0xc443e91ce0, 0xc43e90f1e0, 0xc43e90f200, 0x7fad6035c8e0, 0xc43cbb6660, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43cbb6660, 0xa74a80, 0xc443e91ce0, 0x7fad6035c8e0, 0xc43cbb6660, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43cbb6660, 0xa74a80, 0xc443e91ce0, 0xc43e90f1e0, 0xc43e90f220, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xffffffff, 0x0, 0x19, 0xc442c1f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xc43e90f1e0, 0xc43e90f240, 0xc442c1f878, 0x558012, 0x588a58c8, 0x527f27a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xed01c4fc8, 0x0, 0x0, 0xb19bf)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xc43e90f1e0, 0xc43e90f260, 0xa91b40, 0xc43e90f280, 0xc43e90f1e0, 0xc43e90f280)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xc420246100, 0xc437023ae0, 0xc442c1fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xc43e90f1e0, 0xc43e90f200, 0x8e85, 0xc420318b18, 0x0, 0xc420318b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43564f770, 0xc42fba9aa0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdcb00, 0xc420331380, 0x10265c0, 0xc43564f830, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdcb00, 0xc43564f830)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdcb00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8549 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4241a7530, 0xc4241a7560, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4241a7530, 0xc433c72aa0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4241a7530, 0xc4415dd160, 0x0, 0xc42443bae0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4241a7530, 0xa74a80, 0xc4415dd160, 0x7fad6035c8e0, 0xc4241a7530, 0x1038500, 0xc4241a7500)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4241a7500, 0xa74a80, 0xc4415dd160, 0xc433c72980, 0xc433c729a0, 0x7fad6035c8e0, 0xc4241a7500, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a7500, 0xa74a80, 0xc4415dd160, 0x7fad6035c8e0, 0xc4241a7500, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4241a7500, 0xa74a80, 0xc4415dd160, 0xc433c72980, 0xc433c729c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xffffffff, 0x0, 0x19, 0xc448c2d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xc433c72980, 0xc433c729e0, 0xc448c2d878, 0x558012, 0x588a57b9, 0x38c61ff2)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xed01c4eb9, 0x0, 0x0, 0x2aa54)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xc433c72980, 0xc433c72a00, 0xa91b40, 0xc433c72a20, 0xc433c72980, 0xc433c72a20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xc420246100, 0xc42443b170, 0xc448c2da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xc433c72980, 0xc433c729a0, 0x17d5, 0xc420047b18, 0x0, 0xc420047b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4241a6e40, 0xc42693d740, 0xc420179d60, 0x0, 0x0, 0xc420fa6fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc425cf3700, 0xc420331380, 0x10265c0, 0xc4241a6f00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc425cf3700, 0xc4241a6f00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc425cf3700)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 5575 [select, 13 minutes]:\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc42000d740)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855\n\ngoroutine 5350 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc420489c04)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc420489c00)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x9e919d062dc9c8bf)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc43e303410, 0xc420162418, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43e3030b0, 0xc43e303410, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43e3030b0, 0xc428a42ba0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43e3030b0, 0xc428ccd620, 0x0, 0xc43e30b170, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43e3030b0, 0xa74a80, 0xc428ccd620, 0x7fad6035c8e0, 0xc43e3030b0, 0x1038500, 0xc43e303080)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43e303080, 0xa74a80, 0xc428ccd620, 0xc428a422a0, 0xc428a42360, 0x7fad6035c8e0, 0xc43e303080, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e303080, 0xa74a80, 0xc428ccd620, 0x7fad6035c8e0, 0xc43e303080, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43e303080, 0xa74a80, 0xc428ccd620, 0xc428a422a0, 0xc428a423a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xffffffff, 0x0, 0x19, 0xc440a1d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xc428a422a0, 0xc428a423c0, 0xc440a1d878, 0x558012, 0x588a5760, 0x3efb268)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xed01c4e60, 0x0, 0x0, 0xbdbe9)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xc428a422a0, 0xc428a42400, 0xa91b40, 0xc428a424a0, 0xc428a422a0, 0xc428a424a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xc420246100, 0xc43c501260, 0xc440a1da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xc428a422a0, 0xc428a42360, 0x9464, 0xc420313b18, 0x0, 0xc420313b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43d176840, 0xc4415b5b60, 0xc420179d60, 0x0, 0x0, 0xc42ad71fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0700, 0xc420331380, 0x10265c0, 0xc43d176870, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0700, 0xc43d176870)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0700)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 5328 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc420489c04)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc420489c00)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0xd9c46c7e266cb262)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc4435d9140, 0xc422334c2d, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4435d8c90, 0xc4435d9140, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4435d8c90, 0xc429a08a80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4435d8c90, 0xc42b396440, 0x0, 0xc4435da590, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4435d8c90, 0xa74a80, 0xc42b396440, 0x7fad6035c8e0, 0xc4435d8c90, 0x1038500, 0xc4435d8c60)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4435d8c60, 0xa74a80, 0xc42b396440, 0xc429a08500, 0xc429a08520, 0x7fad6035c8e0, 0xc4435d8c60, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4435d8c60, 0xa74a80, 0xc42b396440, 0x7fad6035c8e0, 0xc4435d8c60, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4435d8c60, 0xa74a80, 0xc42b396440, 0xc429a08500, 0xc429a08540, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xffffffff, 0x0, 0x19, 0xc42d70b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xc429a08500, 0xc429a08560, 0xc42d70b878, 0x558012, 0x588a575f, 0x39ef50b3)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xed01c4e5f, 0x0, 0x0, 0x121526)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xc429a08500, 0xc429a085c0, 0xa91b40, 0xc429a085e0, 0xc429a08500, 0xc429a085e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xc420246100, 0xc443746500, 0xc42d70ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xc429a08500, 0xc429a08520, 0xc307, 0xc420e1ab18, 0x0, 0xc420e1ab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4437407e0, 0xc43bba52c0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb4a00, 0xc420331380, 0x10265c0, 0xc4437409c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb4a00, 0xc4437409c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb4a00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8110 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44151f710, 0xc44151f740, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44151f710, 0xc438f47a40, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44151f710, 0xc442e267e0, 0x0, 0xc434142860, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44151f710, 0xa74a80, 0xc442e267e0, 0x7fad6035c8e0, 0xc44151f710, 0x1038500, 0xc44151f6e0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44151f6e0, 0xa74a80, 0xc442e267e0, 0xc438f47900, 0xc438f47920, 0x7fad6035c8e0, 0xc44151f6e0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151f6e0, 0xa74a80, 0xc442e267e0, 0x7fad6035c8e0, 0xc44151f6e0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44151f6e0, 0xa74a80, 0xc442e267e0, 0xc438f47900, 0xc438f47940, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xffffffff, 0x0, 0x19, 0xc42a6dd7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xc438f47900, 0xc438f47980, 0xc42a6dd878, 0x558012, 0x588a577a, 0xac79db1)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xed01c4e7a, 0x0, 0x0, 0x61ea4)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xc438f47900, 0xc438f479a0, 0xa91b40, 0xc438f479c0, 0xc438f47900, 0xc438f479c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xc420246100, 0xc43f0e50a0, 0xc42a6dda70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xc438f47900, 0xc438f47920, 0x3650, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44151e390, 0xc4234bbbc0, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa700, 0xc420331380, 0x10265c0, 0xc44151e450, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa700, 0xc44151e450)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa700)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9884 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437c0ced0, 0xc437c0cf30, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437c0ced0, 0xc43d1c4840, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437c0ced0, 0xc441054180, 0x0, 0xc438b75ae0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437c0ced0, 0xa74a80, 0xc441054180, 0x7fad6035c8e0, 0xc437c0ced0, 0x1038500, 0xc437c0ce70)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc437c0ce70, 0xa74a80, 0xc441054180, 0xc43d1c4720, 0xc43d1c4740, 0x7fad6035c8e0, 0xc437c0ce70, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c0ce70, 0xa74a80, 0xc441054180, 0x7fad6035c8e0, 0xc437c0ce70, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc437c0ce70, 0xa74a80, 0xc441054180, 0xc43d1c4720, 0xc43d1c4760, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xffffffff, 0x0, 0x19, 0xc4414557e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xc43d1c4720, 0xc43d1c4780, 0xc441455878, 0x558012, 0x588a588b, 0x335a74ca)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xed01c4f8b, 0x0, 0x0, 0x12a002)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xc43d1c4720, 0xc43d1c47a0, 0xa91b40, 0xc43d1c47c0, 0xc43d1c4720, 0xc43d1c47c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xc420246100, 0xc438b97360, 0xc441455a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xc43d1c4720, 0xc43d1c4740, 0x36b6, 0xc42023bb18, 0x0, 0xc42023bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc437c59dd0, 0xc4411972c0, 0xc420179d60, 0x0, 0x0, 0xc439252d40, 0x646e696b)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa300, 0xc420331380, 0x10265c0, 0xc437c59e00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa300, 0xc437c59e00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8886 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42210be00, 0xc42210be30, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42210be00, 0xc4432225c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42210be00, 0xc44a9cf980, 0x0, 0xc4223f7608, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42210be00, 0xa74a80, 0xc44a9cf980, 0x7fad6035c8e0, 0xc42210be00, 0x1038500, 0xc42210bdd0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42210bdd0, 0xa74a80, 0xc44a9cf980, 0xc4432224c0, 0xc4432224e0, 0x7fad6035c8e0, 0xc42210bdd0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210bdd0, 0xa74a80, 0xc44a9cf980, 0x7fad6035c8e0, 0xc42210bdd0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42210bdd0, 0xa74a80, 0xc44a9cf980, 0xc4432224c0, 0xc443222500, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xffffffff, 0x0, 0x19, 0xc4215bb7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xc4432224c0, 0xc443222520, 0xc4215bb878, 0x558012, 0x588a57f2, 0xf03c277)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xed01c4ef2, 0x0, 0x0, 0x6edfc)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xc4432224c0, 0xc443222540, 0xa91b40, 0xc443222560, 0xc4432224c0, 0xc443222560)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xc420246100, 0xc444b25ba0, 0xc4215bba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xc4432224c0, 0xc4432224e0, 0x8be8, 0xc42023db18, 0x0, 0xc42023db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc428928150, 0xc44ac1f3e0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41eab1840)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439c2a200, 0xc420331380, 0x10265c0, 0xc428928570, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439c2a200, 0xc428928570)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439c2a200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9885 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437b7fbf0, 0xc437b7fc20, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437b7fbf0, 0xc449234500, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437b7fbf0, 0xc43d1c5340, 0x0, 0xc438b0ab20, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437b7fbf0, 0xa74a80, 0xc43d1c5340, 0x7fad6035c8e0, 0xc437b7fbf0, 0x1038500, 0xc437b7eb40)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc437b7eb40, 0xa74a80, 0xc43d1c5340, 0xc449234400, 0xc449234420, 0x7fad6035c8e0, 0xc437b7eb40, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437b7eb40, 0xa74a80, 0xc43d1c5340, 0x7fad6035c8e0, 0xc437b7eb40, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc437b7eb40, 0xa74a80, 0xc43d1c5340, 0xc449234400, 0xc449234440, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xffffffff, 0x0, 0x19, 0xc4414577e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xc449234400, 0xc449234460, 0xc441457878, 0x558012, 0x588a588b, 0x374a0585)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xed01c4f8b, 0x0, 0x0, 0x65b8f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xc449234400, 0xc449234480, 0xa91b40, 0xc4492344a0, 0xc449234400, 0xc4492344a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xc420246100, 0xc438b75b90, 0xc441457a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xc449234400, 0xc449234420, 0x2b59, 0xc42023bb18, 0x0, 0xc42023bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc437c00c90, 0xc4411973e0, 0xc420179d60, 0x0, 0x0, 0xc439252d40, 0x646e696b)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa700, 0xc420331380, 0x10265c0, 0xc437c00cf0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa700, 0xc437c00cf0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa700)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8112 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc446b12990, 0xc446b129c0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc446b12990, 0xc438de2d00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc446b12990, 0xc438f22880, 0x0, 0xc446b14740, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc446b12990, 0xa74a80, 0xc438f22880, 0x7fad6035c8e0, 0xc446b12990, 0x1038500, 0xc446b12960)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc446b12960, 0xa74a80, 0xc438f22880, 0xc438de2ba0, 0xc438de2bc0, 0x7fad6035c8e0, 0xc446b12960, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc446b12960, 0xa74a80, 0xc438f22880, 0x7fad6035c8e0, 0xc446b12960, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc446b12960, 0xa74a80, 0xc438f22880, 0xc438de2ba0, 0xc438de2be0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xffffffff, 0x0, 0x19, 0xc43ee437e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xc438de2ba0, 0xc438de2c20, 0xc43ee43878, 0x558012, 0x588a577a, 0xcbdef85)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xed01c4e7a, 0x0, 0x0, 0x4c792)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xc438de2ba0, 0xc438de2c40, 0xa91b40, 0xc438de2c60, 0xc438de2ba0, 0xc438de2c60)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xc420246100, 0xc4341429a0, 0xc43ee43a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xc438de2ba0, 0xc438de2bc0, 0x3da4, 0xc420044b18, 0x0, 0xc420044b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43414e7b0, 0xc4234bbe60, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aaf00, 0xc420331380, 0x10265c0, 0xc43414e870, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aaf00, 0xc43414e870)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aaf00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8153 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc42c13376c)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*WaitGroup).Wait(0xc42c133760)\n\t/usr/local/go/src/sync/waitgroup.go:131 +0x97\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc42c133760, 0xc42c212900)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e\n\ngoroutine 8562 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42244e540, 0xc42244e570, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42244e540, 0xc433c78a00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42244e540, 0xc433cb4160, 0x0, 0xc422165558, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42244e540, 0xa74a80, 0xc433cb4160, 0x7fad6035c8e0, 0xc42244e540, 0x1038500, 0xc42244e510)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42244e510, 0xa74a80, 0xc433cb4160, 0xc433c788e0, 0xc433c78900, 0x7fad6035c8e0, 0xc42244e510, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42244e510, 0xa74a80, 0xc433cb4160, 0x7fad6035c8e0, 0xc42244e510, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42244e510, 0xa74a80, 0xc433cb4160, 0xc433c788e0, 0xc433c78920, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xffffffff, 0x0, 0x19, 0xc448c317e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xc433c788e0, 0xc433c78960, 0xc448c31878, 0x558012, 0x588a57b9, 0x38658113)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xed01c4eb9, 0x0, 0x0, 0x19f243)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xc433c788e0, 0xc433c78980, 0xa91b40, 0xc433c789a0, 0xc433c788e0, 0xc433c789a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xc420246100, 0xc422ecc2e0, 0xc448c31a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xc433c788e0, 0xc433c78900, 0x87ff, 0xc420319b18, 0x0, 0xc420319b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc423344420, 0xc4235d4cc0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5d1149)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd200, 0xc420331380, 0x10265c0, 0xc423344540, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd200, 0xc423344540)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 5364 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204962c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204962c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0xe9d56397ab2f3733)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc441195980, 0xc422334c2d, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441195920, 0xc441195980, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441195920, 0xc429094fc0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441195920, 0xc4292ec3c0, 0x0, 0xc44106b3e0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441195920, 0xa74a80, 0xc4292ec3c0, 0x7fad6035c8e0, 0xc441195920, 0x1038500, 0xc4411958f0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4411958f0, 0xa74a80, 0xc4292ec3c0, 0xc429094c80, 0xc429094cc0, 0x7fad6035c8e0, 0xc4411958f0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4411958f0, 0xa74a80, 0xc4292ec3c0, 0x7fad6035c8e0, 0xc4411958f0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4411958f0, 0xa74a80, 0xc4292ec3c0, 0xc429094c80, 0xc429094ce0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xffffffff, 0x0, 0x19, 0xc42a6e17e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xc429094c80, 0xc429094d40, 0xc42a6e1878, 0x558012, 0x588a5760, 0x26c9b7f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xed01c4e60, 0x0, 0x0, 0x178f91)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xc429094c80, 0xc429094e20, 0xa91b40, 0xc429094e60, 0xc429094c80, 0xc429094e60)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xc420246100, 0xc441865630, 0xc42a6e1a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xc429094c80, 0xc429094cc0, 0x961e, 0xc43d5d2b18, 0x0, 0xc43d5d2b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44132b5c0, 0xc440871b60, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cad400, 0xc420331380, 0x10265c0, 0xc44132b5f0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cad400, 0xc44132b5f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cad400)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9888 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437885110, 0xc437885140, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437885110, 0xc43ce23480, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437885110, 0xc44748ea60, 0x0, 0xc43897fc30, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437885110, 0xa74a80, 0xc44748ea60, 0x7fad6035c8e0, 0xc437885110, 0x1038500, 0xc4378850e0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4378850e0, 0xa74a80, 0xc44748ea60, 0xc43ce23380, 0xc43ce233a0, 0x7fad6035c8e0, 0xc4378850e0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4378850e0, 0xa74a80, 0xc44748ea60, 0x7fad6035c8e0, 0xc4378850e0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4378850e0, 0xa74a80, 0xc44748ea60, 0xc43ce23380, 0xc43ce233c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0xffffffff, 0x0, 0x19, 0xc43d89b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0xc43ce23380, 0xc43ce233e0, 0xc43d89b878, 0x558012, 0x588a588b, 0x3a1d1363)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0x41167e, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0xc43ce23380, 0xc43ce23400, 0xa91b40, 0xc43ce23420, 0xc43ce23380, 0xc43ce23420)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0xc420246100, 0xc4389d41c0, 0xc43d89ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0xc43ce23380, 0xc43ce233a0, 0x5743, 0xc420044b18, 0x0, 0xc420044b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4379569c0, 0xc43c651bc0, 0xc420179d60, 0x0, 0x0, 0xc439252d40, 0x646e696b)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438bfb900, 0xc420331380, 0x10265c0, 0xc437956a20, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438bfb900, 0xc437956a20)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438bfb900)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8267 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc446b0daa0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc444b24d40, 0xc446b0daa0, 0x10343c0, 0xc4201e3040)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326\n\ngoroutine 5580 [select, 13 minutes]:\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc42000d8c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855\n\ngoroutine 11429 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc431de214c)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*WaitGroup).Wait(0xc431de2140)\n\t/usr/local/go/src/sync/waitgroup.go:131 +0x97\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc431de2140, 0xc43c3da7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e\n\ngoroutine 11601 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43b3fd8f0, 0xc43b3fd920, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43b3fd8f0, 0xc43cf35c80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43b3fd8f0, 0xc43d0ea740, 0x0, 0xc42ff18a98, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43b3fd8f0, 0xa74a80, 0xc43d0ea740, 0x7fad6035c8e0, 0xc43b3fd8f0, 0x1038500, 0xc43b3fd8c0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43b3fd8c0, 0xa74a80, 0xc43d0ea740, 0xc43cf35b80, 0xc43cf35ba0, 0x7fad6035c8e0, 0xc43b3fd8c0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b3fd8c0, 0xa74a80, 0xc43d0ea740, 0x7fad6035c8e0, 0xc43b3fd8c0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43b3fd8c0, 0xa74a80, 0xc43d0ea740, 0xc43cf35b80, 0xc43cf35bc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xffffffff, 0x0, 0x19, 0xc43b8337e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xc43cf35b80, 0xc43cf35be0, 0xc43b833878, 0x558012, 0x588a5996, 0x105d0346)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xed01c5096, 0x0, 0x0, 0xb6f82)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xc43cf35b80, 0xc43cf35c00, 0xa91b40, 0xc43cf35c20, 0xc43cf35b80, 0xc43cf35c20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xc420246100, 0xc430195890, 0xc43b833a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xc43cf35b80, 0xc43cf35ba0, 0x7c74, 0xc420e1cb18, 0x0, 0xc420e1cb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc441b0d350, 0xc422873740, 0xc420179d60, 0x0, 0x0, 0x44c49082079f0910, 0x9de395f9d710400f)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438065700, 0xc420331380, 0x10265c0, 0xc441b0d380, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438065700, 0xc441b0d380)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438065700)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10662 [semacquire, 6 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43130bdd0, 0xc43130be00, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43130bdd0, 0xc444079100, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43130bdd0, 0xc4483f0e20, 0x0, 0xc43557ed50, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43130bdd0, 0xa74a80, 0xc4483f0e20, 0x7fad6035c8e0, 0xc43130bdd0, 0x1038500, 0xc43130bda0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43130bda0, 0xa74a80, 0xc4483f0e20, 0xc444079000, 0xc444079020, 0x7fad6035c8e0, 0xc43130bda0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43130bda0, 0xa74a80, 0xc4483f0e20, 0x7fad6035c8e0, 0xc43130bda0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43130bda0, 0xa74a80, 0xc4483f0e20, 0xc444079000, 0xc444079040, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xffffffff, 0x0, 0x19, 0xc4472157e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xc444079000, 0xc444079060, 0xc447215878, 0x558012, 0x588a5921, 0x3a91d9bb)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xed01c5021, 0x0, 0x0, 0x104e94)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xc444079000, 0xc444079080, 0xa91b40, 0xc4440790a0, 0xc444079000, 0xc4440790a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xc420246100, 0xc435730ff0, 0xc447215a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xc444079000, 0xc444079020, 0x88ec, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43138af30, 0xc436c75320, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41edcc21c)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065400, 0xc420331380, 0x10265c0, 0xc43138b080, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065400, 0xc43138b080)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065400)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10659 [semacquire, 6 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4311cd3b0, 0xc4311cd410, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4311cd3b0, 0xc44973b420, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4311cd3b0, 0xc4438e8800, 0x0, 0xc4368f5478, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4311cd3b0, 0xa74a80, 0xc4438e8800, 0x7fad6035c8e0, 0xc4311cd3b0, 0x1038500, 0xc4311cd350)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4311cd350, 0xa74a80, 0xc4438e8800, 0xc43dd3bca0, 0xc43dd3bdc0, 0x7fad6035c8e0, 0xc4311cd350, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4311cd350, 0xa74a80, 0xc4438e8800, 0x7fad6035c8e0, 0xc4311cd350, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4311cd350, 0xa74a80, 0xc4438e8800, 0xc43dd3bca0, 0xc43dd3bec0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xffffffff, 0x0, 0x19, 0xc4322057e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xc43dd3bca0, 0xc43dd3bfc0, 0xc432205878, 0x558012, 0x588a5900, 0xf9d91e7)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xc432205900, 0x0, 0x0, 0x71)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xc43dd3bca0, 0xc44973a040, 0xa91b40, 0xc44973ae20, 0xc43dd3bca0, 0xc44973ae20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xc420246100, 0xc436a82190, 0xc432205a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xc43dd3bca0, 0xc43dd3bdc0, 0x5617, 0xc420314b18, 0x0, 0xc420314b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42da76660, 0xc436014a20, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f0f4464)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390faf00, 0xc420331380, 0x10265c0, 0xc42da766c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390faf00, 0xc42da766c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390faf00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8419 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc42b5b01b0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f\ngithub.com/weaveworks/cortex/ingester.(*Ingester).LabelValues(0xc4201e3040, 0x7fad6035c8e0, 0xc42b5b01b0, 0xc42c6276f0, 0x0, 0xc42c627718, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:298 +0x60\ngithub.com/weaveworks/cortex._Ingester_LabelValues_Handler.func1(0x7fad6035c8e0, 0xc42b5b01b0, 0xa73d60, 0xc42c6276f0, 0x7fad6035c8e0, 0xc42b5b01b0, 0x1038500, 0xc42b5b0180)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:457 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42b5b0180, 0xa73d60, 0xc42c6276f0, 0xc434fcff60, 0xc434fcff80, 0x7fad6035c8e0, 0xc42b5b0180, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b5b0180, 0xa73d60, 0xc42c6276f0, 0x7fad6035c8e0, 0xc42b5b0180, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42b5b0180, 0xa73d60, 0xc42c6276f0, 0xc434fcff60, 0xc434fcffa0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xffffffff, 0x0, 0x19, 0xc4427e57e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xc434fcff60, 0xc434fcffc0, 0xc4427e5878, 0x558012, 0x588a57a4, 0x1650ef3a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xc4427e5900, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xc434fcff60, 0xc434fcffe0, 0xa91b40, 0xc434fcc000, 0xc434fcff60, 0xc434fcc000)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xc420246100, 0xc42c6276e0, 0xc4427e5a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xc434fcff60, 0xc434fcff80, 0xa, 0xc420e1bb18, 0x0, 0xc420e1bb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_LabelValues_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42b5b0060, 0xc42b7fa8a0, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:459 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc42646d600, 0xc420331380, 0x10265f0, 0xc42b5b0120, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc42646d600, 0xc42b5b0120)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc43c1fd850, 0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc42646d600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 11986 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43e5ec630, 0xc43e5ec660, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43e5ec630, 0xc4484c4c80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43e5ec630, 0xc44334b500, 0x0, 0xc432a37310, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43e5ec630, 0xa74a80, 0xc44334b500, 0x7fad6035c8e0, 0xc43e5ec630, 0x1038500, 0xc43e5ec600)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43e5ec600, 0xa74a80, 0xc44334b500, 0xc4484c4b80, 0xc4484c4ba0, 0x7fad6035c8e0, 0xc43e5ec600, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e5ec600, 0xa74a80, 0xc44334b500, 0x7fad6035c8e0, 0xc43e5ec600, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43e5ec600, 0xa74a80, 0xc44334b500, 0xc4484c4b80, 0xc4484c4bc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xffffffff, 0x0, 0x19, 0xc4385a17e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xc4484c4b80, 0xc4484c4be0, 0xc4385a1878, 0x558012, 0x588a59d2, 0x1224f2f3)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xed01c50d2, 0x0, 0x0, 0x185d83)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xc4484c4b80, 0xc4484c4c00, 0xa91b40, 0xc4484c4c20, 0xc4484c4b80, 0xc4484c4c20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xc420246100, 0xc434f81cc0, 0xc4385a1a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xc4484c4b80, 0xc4484c4ba0, 0x7c68, 0xc420239b18, 0x0, 0xc420239b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442452480, 0xc43e3076e0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6200, 0xc420331380, 0x10265c0, 0xc4424524b0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6200, 0xc4424524b0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8650 [chan send, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc430b3e4b0, 0xc436c445a0, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 9664 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44603fef0, 0xc44603ff20, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44603fef0, 0xc449db44a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44603fef0, 0xc43c25a640, 0x0, 0xc441e558d0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44603fef0, 0xa74a80, 0xc43c25a640, 0x7fad6035c8e0, 0xc44603fef0, 0x1038500, 0xc44603fec0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44603fec0, 0xa74a80, 0xc43c25a640, 0xc449db98a0, 0xc449db9a20, 0x7fad6035c8e0, 0xc44603fec0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44603fec0, 0xa74a80, 0xc43c25a640, 0x7fad6035c8e0, 0xc44603fec0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44603fec0, 0xa74a80, 0xc43c25a640, 0xc449db98a0, 0xc449db9ba0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xffffffff, 0x0, 0x19, 0xc42eb3d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xc449db98a0, 0xc449db9d40, 0xc42eb3d878, 0x558012, 0x588a586a, 0x103547fa)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xc42eb3d900, 0x0, 0x0, 0x1)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xc449db98a0, 0xc449db9ea0, 0xa91b40, 0xc449db4020, 0xc449db98a0, 0xc449db4020)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xc420246100, 0xc43ff6bf40, 0xc42eb3da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xc449db98a0, 0xc449db9a20, 0x7de0, 0xc420e1ab18, 0x0, 0xc420e1ab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43dff1050, 0xc445cf84e0, 0xc420179d60, 0x0, 0x0, 0x7fad625dc000, 0xc41eac8542)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87a00, 0xc420331380, 0x10265c0, 0xc43dff1080, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87a00, 0xc43dff1080)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87a00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8209 [chan send, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc4200d4a50, 0xc446b0da40, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 8150 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42c1daf30, 0xc42c1daf60, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42c1daf30, 0xc43a5a11c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42c1daf30, 0xc43a3bce60, 0x0, 0xc42c133610, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42c1daf30, 0xa74a80, 0xc43a3bce60, 0x7fad6035c8e0, 0xc42c1daf30, 0x1038500, 0xc42c1daf00)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c1daf00, 0xa74a80, 0xc43a3bce60, 0xc43a5a10c0, 0xc43a5a10e0, 0x7fad6035c8e0, 0xc42c1daf00, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c1daf00, 0xa74a80, 0xc43a3bce60, 0x7fad6035c8e0, 0xc42c1daf00, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c1daf00, 0xa74a80, 0xc43a3bce60, 0xc43a5a10c0, 0xc43a5a1100, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xffffffff, 0x0, 0x19, 0xc439edf7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xc43a5a10c0, 0xc43a5a1120, 0xc439edf878, 0x558012, 0x588a579c, 0x3dd1ed7)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xed01c4e9c, 0x0, 0x0, 0x88d4b)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xc43a5a10c0, 0xc43a5a1140, 0xa91b40, 0xc43a5a1160, 0xc43a5a10c0, 0xc43a5a1160)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xc420246100, 0xc42d4e0990, 0xc439edfa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xc43a5a10c0, 0xc43a5a10e0, 0x73f7, 0xc42023cb18, 0x0, 0xc42023cb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d4c2e70, 0xc44088fd40, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42646c800, 0xc420331380, 0x10265c0, 0xc42d4c2ea0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42646c800, 0xc42d4c2ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42646c800)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 9880 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437eb4360, 0xc437eb4390, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437eb4360, 0xc441109040, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437eb4360, 0xc44311f6e0, 0x0, 0xc438d77770, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437eb4360, 0xa74a80, 0xc44311f6e0, 0x7fad6035c8e0, 0xc437eb4360, 0x1038500, 0xc437eb4300)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc437eb4300, 0xa74a80, 0xc44311f6e0, 0xc441108f20, 0xc441108f40, 0x7fad6035c8e0, 0xc437eb4300, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437eb4300, 0xa74a80, 0xc44311f6e0, 0x7fad6035c8e0, 0xc437eb4300, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc437eb4300, 0xa74a80, 0xc44311f6e0, 0xc441108f20, 0xc441108f60, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xffffffff, 0x0, 0x19, 0xc44578d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xc441108f20, 0xc441108fa0, 0xc44578d878, 0x558012, 0x588a5888, 0x103f7fdd)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xc44578d900, 0x0, 0x0, 0x7f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xc441108f20, 0xc441108fc0, 0xa91b40, 0xc441108fe0, 0xc441108f20, 0xc441108fe0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xc420246100, 0xc438e98fd0, 0xc44578da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xc441108f20, 0xc441108f40, 0x7b93, 0xc420319b18, 0x0, 0xc420319b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc438262ff0, 0xc4415b5560, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f6f34f0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b4800, 0xc420331380, 0x10265c0, 0xc438263020, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b4800, 0xc438263020)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b4800)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8516 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4223f5860, 0xc4223f5890, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4223f5860, 0xc433bc11c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4223f5860, 0xc434049240, 0x0, 0xc4223f7c20, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4223f5860, 0xa74a80, 0xc434049240, 0x7fad6035c8e0, 0xc4223f5860, 0x1038500, 0xc4223f5830)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4223f5830, 0xa74a80, 0xc434049240, 0xc433bc10a0, 0xc433bc10c0, 0x7fad6035c8e0, 0xc4223f5830, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4223f5830, 0xa74a80, 0xc434049240, 0x7fad6035c8e0, 0xc4223f5830, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4223f5830, 0xa74a80, 0xc434049240, 0xc433bc10a0, 0xc433bc10e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xffffffff, 0x0, 0x19, 0xc43fccf7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xc433bc10a0, 0xc433bc1100, 0xc43fccf878, 0x558012, 0x588a57b9, 0x3a0b8463)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xed01c4eb9, 0x0, 0x0, 0x4b18ff)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xc433bc10a0, 0xc433bc1120, 0xa91b40, 0xc433bc1140, 0xc433bc10a0, 0xc433bc1140)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xc420246100, 0xc426a91790, 0xc43fccfa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xc433bc10a0, 0xc433bc10c0, 0x63b9, 0xc420e18b18, 0x0, 0xc420e18b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4269fd860, 0xc428757440, 0xc420179d60, 0x0, 0x0, 0x4d1279726f746973, 0x6f642e7865646e69)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4ec00, 0xc420331380, 0x10265c0, 0xc4269fd980, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4ec00, 0xc4269fd980)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4ec00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8081 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc422f9ec90, 0xc422f9ecc0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc422f9ec90, 0xc433cc2fe0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc422f9ec90, 0xc433d357c0, 0x0, 0xc422ecc290, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc422f9ec90, 0xa74a80, 0xc433d357c0, 0x7fad6035c8e0, 0xc422f9ec90, 0x1038500, 0xc422f9ec60)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc422f9ec60, 0xa74a80, 0xc433d357c0, 0xc433cc2ea0, 0xc433cc2ec0, 0x7fad6035c8e0, 0xc422f9ec60, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc422f9ec60, 0xa74a80, 0xc433d357c0, 0x7fad6035c8e0, 0xc422f9ec60, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc422f9ec60, 0xa74a80, 0xc433d357c0, 0xc433cc2ea0, 0xc433cc2ee0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xffffffff, 0x0, 0x19, 0xc448f037e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xc433cc2ea0, 0xc433cc2f20, 0xc448f03878, 0x558012, 0x588a57b9, 0x38574e00)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xed01c4eb9, 0x0, 0x0, 0xa931e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xc433cc2ea0, 0xc433cc2f40, 0xa91b40, 0xc433cc2f60, 0xc433cc2ea0, 0xc433cc2f60)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xc420246100, 0xc423602b60, 0xc448f03a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xc433cc2ea0, 0xc433cc2ec0, 0x92b3, 0xc42023cb18, 0x0, 0xc42023cb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc423344180, 0xc4235d41e0, 0xc420179d60, 0x0, 0x0, 0x61632f736f6d6564, 0x36383639653a7472)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd100, 0xc420331380, 0x10265c0, 0xc4233445a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd100, 0xc4233445a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd100)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10274 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4359e5bf0, 0xc4359e5c20, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4359e5bf0, 0xc440825440, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4359e5bf0, 0xc43a753600, 0x0, 0xc4372372a0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4359e5bf0, 0xa74a80, 0xc43a753600, 0x7fad6035c8e0, 0xc4359e5bf0, 0x1038500, 0xc4359e5bc0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4359e5bc0, 0xa74a80, 0xc43a753600, 0xc440825320, 0xc440825340, 0x7fad6035c8e0, 0xc4359e5bc0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4359e5bc0, 0xa74a80, 0xc43a753600, 0x7fad6035c8e0, 0xc4359e5bc0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4359e5bc0, 0xa74a80, 0xc43a753600, 0xc440825320, 0xc440825360, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xffffffff, 0x0, 0x19, 0xc4422737e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xc440825320, 0xc440825380, 0xc442273878, 0x558012, 0x588a58c7, 0x3728a266)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xed01c4fc7, 0x0, 0x0, 0x11d9b1)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xc440825320, 0xc4408253c0, 0xa91b40, 0xc4408253e0, 0xc440825320, 0xc4408253e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xc420246100, 0xc4372de5c0, 0xc442273a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xc440825320, 0xc440825340, 0x4899, 0xc420318b18, 0x0, 0xc420318b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc435b26780, 0xc4320ccb40, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc200, 0xc420331380, 0x10265c0, 0xc435b267e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc200, 0xc435b267e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8651 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4386531dc)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*WaitGroup).Wait(0xc4386531d0)\n\t/usr/local/go/src/sync/waitgroup.go:131 +0x97\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc4386531d0, 0xc436c44600)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e\n\ngoroutine 8721 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc442ded230, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc442ded230, 0x159d0153bd2, 0x159d04d14b2, 0xc42376afe8, 0x1, 0x1, 0xc442ded200, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:272 +0x7e\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc442ded230, 0xc442ded170, 0x0, 0xc442ca6600, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc442ded230, 0xa934c0, 0xc442ded170, 0x7fad6035c8e0, 0xc442ded230, 0x1038500, 0xc442ded200)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc442ded200, 0xa934c0, 0xc442ded170, 0xc432045800, 0xc432045820, 0x7fad6035c8e0, 0xc442ded200, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded200, 0xa934c0, 0xc442ded170, 0x7fad6035c8e0, 0xc442ded200, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc442ded200, 0xa934c0, 0xc442ded170, 0xc432045800, 0xc432045860, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xffffffff, 0x0, 0x19, 0xc431f097e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xc432045800, 0xc4320458a0, 0xc431f09878, 0x558012, 0x588a57df, 0x1570a41a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xed01c4edf, 0x0, 0x0, 0x9743)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xc432045800, 0xc4320458c0, 0xa91b40, 0xc432045900, 0xc432045800, 0xc432045900)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xc420246100, 0xc442ca65c0, 0xc431f09a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xc432045800, 0xc432045820, 0x41, 0xc420e1ab18, 0x0, 0xc420e1ab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442ded050, 0xc43ac10f60, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc420cb9e00, 0xc420331380, 0x10265d8, 0xc442ded110, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc420cb9e00, 0xc442ded110)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc43e3ca700, 0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc420cb9e00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8148 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42751a5a0, 0xc42751a5d0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42751a5a0, 0xc437af2b80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42751a5a0, 0xc437b5e600, 0x0, 0xc4275185f8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42751a5a0, 0xa74a80, 0xc437b5e600, 0x7fad6035c8e0, 0xc42751a5a0, 0x1038500, 0xc42751a570)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42751a570, 0xa74a80, 0xc437b5e600, 0xc437af2a60, 0xc437af2a80, 0x7fad6035c8e0, 0xc42751a570, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42751a570, 0xa74a80, 0xc437b5e600, 0x7fad6035c8e0, 0xc42751a570, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42751a570, 0xa74a80, 0xc437b5e600, 0xc437af2a60, 0xc437af2aa0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xffffffff, 0x0, 0x19, 0xc4454c37e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xc437af2a60, 0xc437af2ac0, 0xc4454c3878, 0x558012, 0x588a579b, 0x36a0864d)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xed01c4e9b, 0x0, 0x0, 0x9d15b)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xc437af2a60, 0xc437af2ae0, 0xa91b40, 0xc437af2b00, 0xc437af2a60, 0xc437af2b00)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xc420246100, 0xc4292d9b20, 0xc4454c3a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xc437af2a60, 0xc437af2a80, 0x6e61, 0xc420e1db18, 0x0, 0xc420e1db08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4292dbef0, 0xc44088ec00, 0xc420179d60, 0x0, 0x0, 0x6c616e7265746e69, 0x412656c020a0a0a)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c200, 0xc420331380, 0x10265c0, 0xc4292dbf20, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c200, 0xc4292dbf20)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10096 [semacquire, 6 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d9a17d0, 0xc42d9a1800, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d9a17d0, 0xc441ed7a60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d9a17d0, 0xc442fa89c0, 0x0, 0xc436a82098, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d9a17d0, 0xa74a80, 0xc442fa89c0, 0x7fad6035c8e0, 0xc42d9a17d0, 0x1038500, 0xc42d9a17a0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d9a17a0, 0xa74a80, 0xc442fa89c0, 0xc441ed7920, 0xc441ed7960, 0x7fad6035c8e0, 0xc42d9a17a0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d9a17a0, 0xa74a80, 0xc442fa89c0, 0x7fad6035c8e0, 0xc42d9a17a0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d9a17a0, 0xa74a80, 0xc442fa89c0, 0xc441ed7920, 0xc441ed7980, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xffffffff, 0x0, 0x19, 0xc432f7d7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xc441ed7920, 0xc441ed79a0, 0xc432f7d878, 0x558012, 0x588a5900, 0xf82dd5f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xed01c5000, 0x0, 0x0, 0x1765b3)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xc441ed7920, 0xc441ed79c0, 0xa91b40, 0xc441ed79e0, 0xc441ed7920, 0xc441ed79e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xc420246100, 0xc436b5cf30, 0xc432f7da70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xc441ed7920, 0xc441ed7960, 0x51d9, 0xc420314b18, 0x0, 0xc420314b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42cd609f0, 0xc436014180, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f0f4464)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fa500, 0xc420331380, 0x10265c0, 0xc42cd60a50, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fa500, 0xc42cd60a50)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fa500)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8802 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc442ded560, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f\ngithub.com/weaveworks/cortex/ingester.(*Ingester).LabelValues(0xc4201e3040, 0x7fad6035c8e0, 0xc442ded560, 0xc442ca66e0, 0x0, 0xc442ca6708, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:298 +0x60\ngithub.com/weaveworks/cortex._Ingester_LabelValues_Handler.func1(0x7fad6035c8e0, 0xc442ded560, 0xa73d60, 0xc442ca66e0, 0x7fad6035c8e0, 0xc442ded560, 0x1038500, 0xc442ded530)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:457 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc442ded530, 0xa73d60, 0xc442ca66e0, 0xc432045b00, 0xc432045b60, 0x7fad6035c8e0, 0xc442ded530, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded530, 0xa73d60, 0xc442ca66e0, 0x7fad6035c8e0, 0xc442ded530, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc442ded530, 0xa73d60, 0xc442ca66e0, 0xc432045b00, 0xc432045b80, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xffffffff, 0x0, 0x19, 0xc431f0f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xc432045b00, 0xc432045ba0, 0xc431f0f878, 0x558012, 0x588a57e0, 0x1e22dc5e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xed01c4ee0, 0x0, 0x0, 0xcc22)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xc432045b00, 0xc432045c00, 0xa91b40, 0xc432045c20, 0xc432045b00, 0xc432045c20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xc420246100, 0xc442ca66d0, 0xc431f0fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xc432045b00, 0xc432045b60, 0xa, 0xc420e1ab18, 0x0, 0xc420e1ab08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_LabelValues_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442ded410, 0xc43ac11080, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:459 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc420cac500, 0xc420331380, 0x10265f0, 0xc442ded4d0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc420cac500, 0xc442ded4d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc440f5ebb0, 0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc420cac500)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 11989 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc439a3be90, 0xc439a3bec0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc439a3be90, 0xc439187e00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc439a3be90, 0xc43e464d20, 0x0, 0xc42f41d450, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc439a3be90, 0xa74a80, 0xc43e464d20, 0x7fad6035c8e0, 0xc439a3be90, 0x1038500, 0xc439a3be60)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc439a3be60, 0xa74a80, 0xc43e464d20, 0xc439187ce0, 0xc439187d00, 0x7fad6035c8e0, 0xc439a3be60, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc439a3be60, 0xa74a80, 0xc43e464d20, 0x7fad6035c8e0, 0xc439a3be60, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc439a3be60, 0xa74a80, 0xc43e464d20, 0xc439187ce0, 0xc439187d40, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xffffffff, 0x0, 0x19, 0xc4376b37e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xc439187ce0, 0xc439187d60, 0xc4376b3878, 0x558012, 0x588a59d6, 0x9570e8)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xed01c50d6, 0x0, 0x0, 0xffe49)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xc439187ce0, 0xc439187d80, 0xa91b40, 0xc439187da0, 0xc439187ce0, 0xc439187da0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xc420246100, 0xc42f54d490, 0xc4376b3a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xc439187ce0, 0xc439187d00, 0x6857, 0xc420e1fb18, 0x0, 0xc420e1fb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442a1cc30, 0xc44103cae0, 0xc420179d60, 0x0, 0x0, 0x3066732d36303736, 0x656c020a090a3479)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4394f3000, 0xc420331380, 0x10265c0, 0xc442a1cc60, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4394f3000, 0xc442a1cc60)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4394f3000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 11291 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42943ce70, 0xc42943cf30, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42943ce70, 0xc43dd1f740, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42943ce70, 0xc449760020, 0x0, 0xc432cf33b0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42943ce70, 0xa74a80, 0xc449760020, 0x7fad6035c8e0, 0xc42943ce70, 0x1038500, 0xc429465fb0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc429465fb0, 0xa74a80, 0xc449760020, 0xc43dd1f620, 0xc43dd1f640, 0x7fad6035c8e0, 0xc429465fb0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429465fb0, 0xa74a80, 0xc449760020, 0x7fad6035c8e0, 0xc429465fb0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc429465fb0, 0xa74a80, 0xc449760020, 0xc43dd1f620, 0xc43dd1f660, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xffffffff, 0x0, 0x19, 0xc437c7b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xc43dd1f620, 0xc43dd1f680, 0xc437c7b878, 0x558012, 0x588a5978, 0x12cd7b59)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xed01c5078, 0x0, 0x0, 0xb4e72)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xc43dd1f620, 0xc43dd1f6a0, 0xa91b40, 0xc43dd1f6e0, 0xc43dd1f620, 0xc43dd1f6e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xc420246100, 0xc432dd5da0, 0xc437c7ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xc43dd1f620, 0xc43dd1f640, 0x79e2, 0xc420e18b18, 0x0, 0xc420e18b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a29a480, 0xc442bd43c0, 0xc420179d60, 0x0, 0x0, 0x6236633266303862, 0x3462316364333339)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa300, 0xc420331380, 0x10265c0, 0xc4298885d0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa300, 0xc4298885d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa300)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 5579 [IO wait]:\nnet.runtime_pollWait(0x7fad6039cfa0, 0x72, 0xb)\n\t/usr/local/go/src/runtime/netpoll.go:160 +0x59\nnet.(*pollDesc).wait(0xc43d2bfc60, 0x72, 0xc4202388d8, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38\nnet.(*pollDesc).waitRead(0xc43d2bfc60, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34\nnet.(*netFD).Read(0xc43d2bfc00, 0xc43e3d2000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)\n\t/usr/local/go/src/net/fd_unix.go:243 +0x1a1\nnet.(*conn).Read(0xc43e50a018, 0xc43e3d2000, 0x8000, 0x8000, 0x0, 0x0, 0x0)\n\t/usr/local/go/src/net/net.go:173 +0x70\nbufio.(*Reader).fill(0xc445d837a0)\n\t/usr/local/go/src/bufio/bufio.go:97 +0x10c\nbufio.(*Reader).Read(0xc445d837a0, 0xc42000d838, 0x9, 0x9, 0x43fe80, 0xc42c6149f0, 0x0)\n\t/usr/local/go/src/bufio/bufio.go:209 +0x1bc\nio.ReadAtLeast(0x102f380, 0xc445d837a0, 0xc42000d838, 0x9, 0x9, 0x9, 0xc420238ae8, 0x451a90, 0xc441066c80)\n\t/usr/local/go/src/io/io.go:307 +0xa4\nio.ReadFull(0x102f380, 0xc445d837a0, 0xc42000d838, 0x9, 0x9, 0xc42c6149f0, 0x0, 0x1)\n\t/usr/local/go/src/io/io.go:325 +0x58\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc42000d838, 0x9, 0x9, 0x102f380, 0xc445d837a0, 0x0, 0xc400000000, 0xc420238c80, 0x7f3105)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b\ngithub.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42000d800, 0xf, 0xf, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc43e509710, 0xc44283d9e0, 0xc44283d9e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42000d8c0, 0xc43e5097a0, 0xb475b8)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc42000d8c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc43e50a018, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc43e50a018)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac\n\ngoroutine 11600 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441b0c000, 0xc441b0c030, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441b0c000, 0xc43f89b960, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441b0c000, 0xc43d343f80, 0x0, 0xc430195480, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441b0c000, 0xa74a80, 0xc43d343f80, 0x7fad6035c8e0, 0xc441b0c000, 0x1038500, 0xc43cdedfb0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43cdedfb0, 0xa74a80, 0xc43d343f80, 0xc43f89b840, 0xc43f89b860, 0x7fad6035c8e0, 0xc43cdedfb0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43cdedfb0, 0xa74a80, 0xc43d343f80, 0x7fad6035c8e0, 0xc43cdedfb0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43cdedfb0, 0xa74a80, 0xc43d343f80, 0xc43f89b840, 0xc43f89b8a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xffffffff, 0x0, 0x19, 0xc4310cb7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xc43f89b840, 0xc43f89b8c0, 0xc4310cb878, 0x558012, 0x588a5996, 0x104a2b66)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xed01c5096, 0x0, 0x0, 0xc9d51)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xc43f89b840, 0xc43f89b8e0, 0xa91b40, 0xc43f89b900, 0xc43f89b840, 0xc43f89b900)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xc420246100, 0xc4303977e0, 0xc4310cba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xc43f89b840, 0xc43f89b860, 0x79a4, 0xc420e1cb18, 0x0, 0xc420e1cb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc445745b00, 0xc425c87a40, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ef31943)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065000, 0xc420331380, 0x10265c0, 0xc445745b30, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065000, 0xc445745b30)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8553 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4386526ec)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*WaitGroup).Wait(0xc4386526e0)\n\t/usr/local/go/src/sync/waitgroup.go:131 +0x97\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc4386526e0, 0xc42e9d8900)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e\n\ngoroutine 11216 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42aef1050, 0xc42aef10b0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42aef1050, 0xc4444f4080, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42aef1050, 0xc444da8a40, 0x0, 0xc43523fb50, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42aef1050, 0xa74a80, 0xc444da8a40, 0x7fad6035c8e0, 0xc42aef1050, 0x1038500, 0xc42aef0f90)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42aef0f90, 0xa74a80, 0xc444da8a40, 0xc42d41bf60, 0xc42d41bf80, 0x7fad6035c8e0, 0xc42aef0f90, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42aef0f90, 0xa74a80, 0xc444da8a40, 0x7fad6035c8e0, 0xc42aef0f90, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42aef0f90, 0xa74a80, 0xc444da8a40, 0xc42d41bf60, 0xc42d41bfa0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xffffffff, 0x0, 0x19, 0xc42805b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xc42d41bf60, 0xc42d41bfc0, 0xc42805b878, 0x558012, 0x588a595d, 0x3a9b5973)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xc42805b900, 0x0, 0x0, 0x1)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xc42d41bf60, 0xc42d41bfe0, 0xa91b40, 0xc4444f4020, 0xc42d41bf60, 0xc4444f4020)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xc420246100, 0xc4352bb640, 0xc42805ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xc42d41bf60, 0xc42d41bf80, 0x6f18, 0xc440d64b18, 0x0, 0xc440d64b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42fd7c5a0, 0xc43dd27c80, 0xc420179d60, 0x0, 0x0, 0xc434e02fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7100, 0xc420331380, 0x10265c0, 0xc42fd7c600, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7100, 0xc42fd7c600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7100)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10273 [semacquire, 8 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc435ad2390, 0xc435ad23f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc435ad2390, 0xc43a752b80, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc435ad2390, 0xc43f2e5fc0, 0x0, 0xc4372de548, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc435ad2390, 0xa74a80, 0xc43f2e5fc0, 0x7fad6035c8e0, 0xc435ad2390, 0x1038500, 0xc435ad2330)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc435ad2330, 0xa74a80, 0xc43f2e5fc0, 0xc43a73fee0, 0xc43a73ff20, 0x7fad6035c8e0, 0xc435ad2330, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435ad2330, 0xa74a80, 0xc43f2e5fc0, 0x7fad6035c8e0, 0xc435ad2330, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc435ad2330, 0xa74a80, 0xc43f2e5fc0, 0xc43a73fee0, 0xc43a73ff40, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xffffffff, 0x0, 0x19, 0xc4422717e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xc43a73fee0, 0xc43a73ff60, 0xc442271878, 0x558012, 0x588a58c7, 0x371d1501)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xc442271900, 0x0, 0x0, 0x57)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xc43a73fee0, 0xc43a752ae0, 0xa91b40, 0xc43a752b20, 0xc43a73fee0, 0xc43a752b20)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xc420246100, 0xc4373a70b0, 0xc442271a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xc43a73fee0, 0xc43a73ff20, 0x3c47, 0xc420239b18, 0x0, 0xc420239b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc435b26210, 0xc4325b32c0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc100, 0xc420331380, 0x10265c0, 0xc435b26270, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc100, 0xc435b26270)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc100)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 12731 [semacquire]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43517bbf0, 0xc43517bc20, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43517bbf0, 0xc446896800, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43517bbf0, 0xc44634a520, 0x0, 0xc428ac99b0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43517bbf0, 0xa74a80, 0xc44634a520, 0x7fad6035c8e0, 0xc43517bbf0, 0x1038500, 0xc43517bbc0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43517bbc0, 0xa74a80, 0xc44634a520, 0xc4468966e0, 0xc446896700, 0x7fad6035c8e0, 0xc43517bbc0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43517bbc0, 0xa74a80, 0xc44634a520, 0x7fad6035c8e0, 0xc43517bbc0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43517bbc0, 0xa74a80, 0xc44634a520, 0xc4468966e0, 0xc446896720, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xffffffff, 0x0, 0x19, 0xc43530f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xc4468966e0, 0xc446896740, 0xc43530f878, 0x558012, 0x588a5a4a, 0x11c7020f)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xed01c514a, 0x0, 0x0, 0x9ee520)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xc4468966e0, 0xc446896760, 0xa91b40, 0xc4468967a0, 0xc4468966e0, 0xc4468967a0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xc420246100, 0xc42ec33740, 0xc43530fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xc4468966e0, 0xc446896700, 0x788a, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43c8273b0, 0xc440cd2720, 0xc420179d60, 0x0, 0x0, 0x10, 0xc437c4ef58)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7200, 0xc420331380, 0x10265c0, 0xc43c8273e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7200, 0xc43c8273e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 11292 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc429dd08a0, 0xc429dd0900, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc429dd08a0, 0xc43c0a46e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc429dd08a0, 0xc43f3a82a0, 0x0, 0xc432fa2c30, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc429dd08a0, 0xa74a80, 0xc43f3a82a0, 0x7fad6035c8e0, 0xc429dd08a0, 0x1038500, 0xc429dd07e0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc429dd07e0, 0xa74a80, 0xc43f3a82a0, 0xc43c0a45c0, 0xc43c0a45e0, 0x7fad6035c8e0, 0xc429dd07e0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429dd07e0, 0xa74a80, 0xc43f3a82a0, 0x7fad6035c8e0, 0xc429dd07e0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc429dd07e0, 0xa74a80, 0xc43f3a82a0, 0xc43c0a45c0, 0xc43c0a4600, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xffffffff, 0x0, 0x19, 0xc437c7f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xc43c0a45c0, 0xc43c0a4620, 0xc437c7f878, 0x558012, 0x588a5978, 0x12ac6d23)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xed01c5078, 0x0, 0x0, 0xca46b)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xc43c0a45c0, 0xc43c0a4640, 0xa91b40, 0xc43c0a4660, 0xc43c0a45c0, 0xc43c0a4660)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xc420246100, 0xc4331ed550, 0xc437c7fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xc43c0a45c0, 0xc43c0a45e0, 0x7876, 0xc420e1fb18, 0x0, 0xc420e1fb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a29aab0, 0xc442bd4300, 0xc420179d60, 0x0, 0x0, 0x65646f6e040a230a, 0x3237312d70691b12)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa400, 0xc420331380, 0x10265c0, 0xc42a2586c0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa400, 0xc42a2586c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa400)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 10457 [chan receive, 6 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc448f6d860, 0xc42c051590)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47\nnet/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc448f6d860, 0xc42c051590)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc448f6d860, 0xc42c051590)\n\t/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc44873be20, 0xc42c051590)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab\nnet/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc44873be20, 0xc42c051590)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc44873be20, 0xc42c051590)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc44873be00, 0xc42c0513b0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2\nnet/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc44873be00, 0xc42c0513b0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc44873bde0, 0xc42c0513b0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c\nnet/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc44873bde0, 0xc42c0513b0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\ngithub.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4493e71e0, 0xc42c0513b0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449\nnet/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4493e71e0, 0xc42c0512c0)\n\t/usr/local/go/src/net/http/server.go:1726 +0x44\nnet/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4493e71e0, 0xc42c0512c0)\n\t/usr/local/go/src/net/http/server.go:2202 +0x7d\nnet/http.(*conn).serve(0xc430c1be80, 0x1038440, 0xc441cbf1c0)\n\t/usr/local/go/src/net/http/server.go:1579 +0x4b7\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:2293 +0x44d\n\ngoroutine 11419 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc447de05d0, 0xc447de0600, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc447de05d0, 0xc442adc980, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc447de05d0, 0xc43e0c0600, 0x0, 0xc431f190b8, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc447de05d0, 0xa74a80, 0xc43e0c0600, 0x7fad6035c8e0, 0xc447de05d0, 0x1038500, 0xc447de05a0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc447de05a0, 0xa74a80, 0xc43e0c0600, 0xc442adc860, 0xc442adc880, 0x7fad6035c8e0, 0xc447de05a0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447de05a0, 0xa74a80, 0xc43e0c0600, 0x7fad6035c8e0, 0xc447de05a0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc447de05a0, 0xa74a80, 0xc43e0c0600, 0xc442adc860, 0xc442adc8a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xffffffff, 0x0, 0x19, 0xc4361897e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xc442adc860, 0xc442adc8c0, 0xc436189878, 0x558012, 0x588a597c, 0x47ef91)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xed01c507c, 0x0, 0x0, 0x99565)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xc442adc860, 0xc442adc8e0, 0xa91b40, 0xc442adc900, 0xc442adc860, 0xc442adc900)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xc420246100, 0xc431f9b760, 0xc436189a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xc442adc860, 0xc442adc880, 0x4bc6, 0xc420318b18, 0x0, 0xc420318b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc440bce8a0, 0xc439a5dce0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ed5c013)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438723200, 0xc420331380, 0x10265c0, 0xc440bce8d0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438723200, 0xc440bce8d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438723200)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 12080 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc442d01dd0, 0xc442d01e00, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc442d01dd0, 0xc441f79ae0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc442d01dd0, 0xc442f320e0, 0x0, 0xc428dc6280, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc442d01dd0, 0xa74a80, 0xc442f320e0, 0x7fad6035c8e0, 0xc442d01dd0, 0x1038500, 0xc442d01da0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc442d01da0, 0xa74a80, 0xc442f320e0, 0xc441f799e0, 0xc441f79a00, 0x7fad6035c8e0, 0xc442d01da0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d01da0, 0xa74a80, 0xc442f320e0, 0x7fad6035c8e0, 0xc442d01da0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc442d01da0, 0xa74a80, 0xc442f320e0, 0xc441f799e0, 0xc441f79a20, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xffffffff, 0x0, 0x19, 0xc435e097e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xc441f799e0, 0xc441f79a40, 0xc435e09878, 0x558012, 0x588a59f3, 0x37b6149d)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xed01c50f3, 0x0, 0x0, 0x73c98)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xc441f799e0, 0xc441f79a60, 0xa91b40, 0xc441f79a80, 0xc441f799e0, 0xc441f79a80)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xc420246100, 0xc4295c22c0, 0xc435e09a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xc441f799e0, 0xc441f79a00, 0x472f, 0xc435e50b18, 0x0, 0xc435e50b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442d001b0, 0xc4445a80c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41fa28d4b)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438064000, 0xc420331380, 0x10265c0, 0xc442d001e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438064000, 0xc442d001e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438064000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 12391 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc439e9b2c0, 0xc439e9b2f0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc439e9b2c0, 0xc4414e1860, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc439e9b2c0, 0xc442b7e000, 0x0, 0xc439e71070, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc439e9b2c0, 0xa74a80, 0xc442b7e000, 0x7fad6035c8e0, 0xc439e9b2c0, 0x1038500, 0xc439e9b290)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc439e9b290, 0xa74a80, 0xc442b7e000, 0xc4414e1760, 0xc4414e1780, 0x7fad6035c8e0, 0xc439e9b290, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc439e9b290, 0xa74a80, 0xc442b7e000, 0x7fad6035c8e0, 0xc439e9b290, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc439e9b290, 0xa74a80, 0xc442b7e000, 0xc4414e1760, 0xc4414e17a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xffffffff, 0x0, 0x19, 0xc4353677e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xc4414e1760, 0xc4414e17c0, 0xc435367878, 0x558012, 0x588a5a0e, 0x11594457)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xc435367900, 0x0, 0x0, 0xbf)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xc4414e1760, 0xc4414e17e0, 0xa91b40, 0xc4414e1800, 0xc4414e1760, 0xc4414e1800)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xc420246100, 0xc43a105c40, 0xc435367a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xc4414e1760, 0xc4414e1780, 0x7c58, 0xc420049b18, 0x0, 0xc420049b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43be19320, 0xc4377bdbc0, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1c00, 0xc420331380, 0x10265c0, 0xc43be19350, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1c00, 0xc43be19350)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1c00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8517 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42e940180, 0xc42e9401b0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42e940180, 0xc4405a54e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42e940180, 0xc433c72ec0, 0x0, 0xc42e9daaf0, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42e940180, 0xa74a80, 0xc433c72ec0, 0x7fad6035c8e0, 0xc42e940180, 0x1038500, 0xc42e940150)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42e940150, 0xa74a80, 0xc433c72ec0, 0xc4405a53e0, 0xc4405a5400, 0x7fad6035c8e0, 0xc42e940150, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e940150, 0xa74a80, 0xc433c72ec0, 0x7fad6035c8e0, 0xc42e940150, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42e940150, 0xa74a80, 0xc433c72ec0, 0xc4405a53e0, 0xc4405a5420, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xffffffff, 0x0, 0x19, 0xc448c2f7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xc4405a53e0, 0xc4405a5440, 0xc448c2f878, 0x558012, 0x588a57b9, 0x3b1ba526)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xed01c4eb9, 0x0, 0x0, 0xd01ee)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xc4405a53e0, 0xc4405a5460, 0xa91b40, 0xc4405a5480, 0xc4405a53e0, 0xc4405a5480)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xc420246100, 0xc42443bbb0, 0xc448c2fa70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xc4405a53e0, 0xc4405a5400, 0x7a83, 0xc420e18b18, 0x0, 0xc420e18b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42210a6c0, 0xc42693db00, 0xc420179d60, 0x0, 0x0, 0x4d1279726f746973, 0x6f642e7865646e69)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422b4f000, 0xc420331380, 0x10265c0, 0xc42210a6f0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422b4f000, 0xc42210a6f0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422b4f000)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 12198 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4231ee37c)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*WaitGroup).Wait(0xc4231ee370)\n\t/usr/local/go/src/sync/waitgroup.go:131 +0x97\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc4231ee370, 0xc4445a86c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e\n\ngoroutine 12197 [chan send, 2 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc43cb882d0, 0xc4445a8660, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 8111 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc440b30930, 0xc440b30960, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc440b30930, 0xc438ec47e0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc440b30930, 0xc43900d1e0, 0x0, 0xc440198100, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc440b30930, 0xa74a80, 0xc43900d1e0, 0x7fad6035c8e0, 0xc440b30930, 0x1038500, 0xc440b30900)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc440b30900, 0xa74a80, 0xc43900d1e0, 0xc438ec46a0, 0xc438ec46c0, 0x7fad6035c8e0, 0xc440b30900, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440b30900, 0xa74a80, 0xc43900d1e0, 0x7fad6035c8e0, 0xc440b30900, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc440b30900, 0xa74a80, 0xc43900d1e0, 0xc438ec46a0, 0xc438ec4700, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xffffffff, 0x0, 0x19, 0xc4200597e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xc438ec46a0, 0xc438ec4720, 0xc420059878, 0x558012, 0x588a577a, 0xc811a11)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xed01c4e7a, 0x0, 0x0, 0x952a7)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xc438ec46a0, 0xc438ec4740, 0xa91b40, 0xc438ec4760, 0xc438ec46a0, 0xc438ec4760)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xc420246100, 0xc441a503a0, 0xc420059a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xc438ec46a0, 0xc438ec46c0, 0x3e9c, 0xc420e1eb18, 0x0, 0xc420e1eb08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43414e5a0, 0xc43b0b85a0, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aae00, 0xc420331380, 0x10265c0, 0xc43414e5d0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aae00, 0xc43414e5d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aae00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 11428 [chan send, 4 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc441ff1b30, 0xc43c3da780, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 5347 [semacquire, 13 minutes]:\nsync.runtime_Semacquire(0xc4204895c4)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*Mutex).Lock(0xc4204895c0)\n\t/usr/local/go/src/sync/mutex.go:85 +0xd0\ngithub.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x475bf267ffed65dd)\n\t/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e\ngithub.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc44499fbf0, 0xc420162418, 0x1)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44499f740, 0xc44499fbf0, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44499f740, 0xc42b418560, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44499f740, 0xc42b4a0820, 0x0, 0xc4446e8c10, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44499f740, 0xa74a80, 0xc42b4a0820, 0x7fad6035c8e0, 0xc44499f740, 0x1038500, 0xc44499f710)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44499f710, 0xa74a80, 0xc42b4a0820, 0xc42b435d60, 0xc42b435d80, 0x7fad6035c8e0, 0xc44499f710, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44499f710, 0xa74a80, 0xc42b4a0820, 0x7fad6035c8e0, 0xc44499f710, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44499f710, 0xa74a80, 0xc42b4a0820, 0xc42b435d60, 0xc42b435e00, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0xffffffff, 0x0, 0x19, 0xc4344eb7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0xc42b435d60, 0xc42b435e20, 0xc4344eb878, 0x558012, 0x588a575f, 0x36e7dde4)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0x411715, 0x0, 0x0, 0x200)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0xc42b435d60, 0xc42b435ea0, 0xa91b40, 0xc42b435ec0, 0xc42b435d60, 0xc42b435ec0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0xc420246100, 0xc444aea970, 0xc4344eba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0xc42b435d60, 0xc42b435d80, 0xad5a, 0xc420313b18, 0x0, 0xc420313b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc444ae7500, 0xc448c5d800, 0xc420179d60, 0x0, 0x0, 0xc42ad71fa8, 0x4b2ba1)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbf400, 0xc420331380, 0x10265c0, 0xc444ae7530, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbf400, 0xc444ae7530)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbf400)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8127 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc43d97727c)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*WaitGroup).Wait(0xc43d977270)\n\t/usr/local/go/src/sync/waitgroup.go:131 +0x97\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc43d977270, 0xc43b0b95c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e\n\ngoroutine 8567 [semacquire, 4 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4468dad20, 0xc4468dad50, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4468dad20, 0xc43b31a0a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4468dad20, 0xc43fb024e0, 0x0, 0xc42de30c50, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4468dad20, 0xa74a80, 0xc43fb024e0, 0x7fad6035c8e0, 0xc4468dad20, 0x1038500, 0xc4468dacf0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4468dacf0, 0xa74a80, 0xc43fb024e0, 0xc44272f620, 0xc44272f640, 0x7fad6035c8e0, 0xc4468dacf0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4468dacf0, 0xa74a80, 0xc43fb024e0, 0x7fad6035c8e0, 0xc4468dacf0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4468dacf0, 0xa74a80, 0xc43fb024e0, 0xc44272f620, 0xc44272f660, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xffffffff, 0x0, 0x19, 0xc439e2b7e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xc44272f620, 0xc43b31a000, 0xc439e2b878, 0x558012, 0x588a59b8, 0x38bc914)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xc439e2b900, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xc44272f620, 0xc43b31a020, 0xa91b40, 0xc43b31a040, 0xc44272f620, 0xc43b31a040)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xc420246100, 0xc42ec40bd0, 0xc439e2ba70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xc44272f620, 0xc44272f640, 0x4638, 0xc420314b18, 0x0, 0xc420314b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc440461680, 0xc44847ade0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f600, 0xc420331380, 0x10265c0, 0xc4404616b0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f600, 0xc4404616b0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8291 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc42a8fd2f0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f\ngithub.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42a8fd2f0, 0x159dc2dc318, 0x159dc659bf8, 0xc42c2482f0, 0x1, 0x1, 0xc42a8fd200, 0x1039120, ...)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:272 +0x7e\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42a8fd2f0, 0xc42a8fd230, 0x0, 0xc43c598a50, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f\ngithub.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42a8fd2f0, 0xa934c0, 0xc42a8fd230, 0x7fad6035c8e0, 0xc42a8fd2f0, 0x1038500, 0xc42a8fd2c0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a8fd2c0, 0xa934c0, 0xc42a8fd230, 0xc438220aa0, 0xc438220ac0, 0x7fad6035c8e0, 0xc42a8fd2c0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fd2c0, 0xa934c0, 0xc42a8fd230, 0x7fad6035c8e0, 0xc42a8fd2c0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a8fd2c0, 0xa934c0, 0xc42a8fd230, 0xc438220aa0, 0xc438220ae0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xffffffff, 0x0, 0x19, 0xc4457777e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xc438220aa0, 0xc438220b20, 0xc445777878, 0x558012, 0x588a5791, 0x11ffde7e)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xed01c4e91, 0x0, 0x0, 0x93f5)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xc438220aa0, 0xc438220b40, 0xa91b40, 0xc438220b60, 0xc438220aa0, 0xc438220b60)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xc420246100, 0xc43c598a10, 0xc445777a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xc438220aa0, 0xc438220ac0, 0x41, 0xc420313b18, 0x0, 0xc420313b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a8fd110, 0xc442ba8b40, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc4385fac00, 0xc420331380, 0x10265d8, 0xc42a8fd1d0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc4385fac00, 0xc42a8fd1d0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc43c1fd850, 0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc4385fac00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 11988 [semacquire, 2 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc426bbdef0, 0xc426bbdf20, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc426bbdef0, 0xc43ee83220, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc426bbdef0, 0xc43ee82060, 0x0, 0xc4309ddd70, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc426bbdef0, 0xa74a80, 0xc43ee82060, 0x7fad6035c8e0, 0xc426bbdef0, 0x1038500, 0xc426bbdec0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc426bbdec0, 0xa74a80, 0xc43ee82060, 0xc43ee83120, 0xc43ee83140, 0x7fad6035c8e0, 0xc426bbdec0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426bbdec0, 0xa74a80, 0xc43ee82060, 0x7fad6035c8e0, 0xc426bbdec0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc426bbdec0, 0xa74a80, 0xc43ee82060, 0xc43ee83120, 0xc43ee83160, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xffffffff, 0x0, 0x19, 0xc4377e17e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xc43ee83120, 0xc43ee83180, 0xc4377e1878, 0x558012, 0x588a59d5, 0x382e500c)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xed01c50d5, 0x0, 0x0, 0x86ac6)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xc43ee83120, 0xc43ee831a0, 0xa91b40, 0xc43ee831c0, 0xc43ee83120, 0xc43ee831c0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xc420246100, 0xc431261350, 0xc4377e1a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xc43ee83120, 0xc43ee83140, 0xd89, 0xc440d67b18, 0x0, 0xc440d67b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d3a5ce0, 0xc43d51a240, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2600, 0xc420331380, 0x10265c0, 0xc42d3a5e00, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2600, 0xc42d3a5e00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2600)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8126 [chan send, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc441ff04b0, 0xc43b0b9560, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 8152 [chan send, 12 minutes]:\ngithub.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc42c0502d0, 0xc42c2128a0, 0x44a)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151\ncreated by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a\n\ngoroutine 8864 [semacquire, 10 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4487ec240, 0xc4487ec270, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4487ec240, 0xc44a3d32a0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4487ec240, 0xc44a514ca0, 0x0, 0xc4402dc670, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4487ec240, 0xa74a80, 0xc44a514ca0, 0x7fad6035c8e0, 0xc4487ec240, 0x1038500, 0xc4487ec210)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4487ec210, 0xa74a80, 0xc44a514ca0, 0xc44a3d31a0, 0xc44a3d31c0, 0x7fad6035c8e0, 0xc4487ec210, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4487ec210, 0xa74a80, 0xc44a514ca0, 0x7fad6035c8e0, 0xc4487ec210, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4487ec210, 0xa74a80, 0xc44a514ca0, 0xc44a3d31a0, 0xc44a3d31e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xffffffff, 0x0, 0x19, 0xc444c057e0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xc44a3d31a0, 0xc44a3d3200, 0xc444c05878, 0x558012, 0x588a57f6, 0x42dfb09)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xed01c4ef6, 0x0, 0x0, 0xd2792)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xc44a3d31a0, 0xc44a3d3220, 0xa91b40, 0xc44a3d3240, 0xc44a3d31a0, 0xc44a3d3240)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xc420246100, 0xc449d20670, 0xc444c05a70, 0x410558)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xc44a3d31a0, 0xc44a3d31c0, 0x4d2c, 0xc420313b18, 0x0, 0xc420313b08)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118\ngithub.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442487200, 0xc428978de0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f1a9759)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87f00, 0xc420331380, 0x10265c0, 0xc442487230, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87f00, 0xc442487230)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4\ngithub.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87f00)\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab\ncreated by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1\n\t/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3\n\ngoroutine 8338 [semacquire, 12 minutes]:\nsync.runtime_Semacquire(0xc4201e3084)\n\t/usr/local/go/src/runtime/sema.go:47 +0x30\nsync.(*RWMutex).RLock(0xc4201e3078)\n\t/usr/local/go/src/sync/rwmutex.go:43 +0x5d\ngithub.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d1ab800, 0xc42d1ab830, 0x0, 0x0, 0x0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad\ngithub.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1ab800, 0xc437d58ae0, 0x0, 0x0)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208\ngithub.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1ab800, 0xc437da1440, 0x0, 0xc42b9ddd18, 0xa01e00)\n\t/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79\ngithub.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d1ab800, 0xa74a80, 0xc437da1440, 0x7fad6035c8e0, 0xc42d1ab800, 0x1038500, 0xc42d1ab7d0)\n\t/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2\ngithub.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d1ab7d0, 0xa74a80, 0xc437da1440, 0xc437d589e0, 0xc437d58a00, 0x7fad6035c8e0, 0xc42d1ab7d0, 0xa3e980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1ab7d0, 0xa74a80, 0xc437da1440, 0x7fad6035c8e0, 0xc42d1ab7d0, 0x103b980, 0x1074250)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66\ngithub.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d1ab7d0, 0xa74a80, 0xc437da1440, 0xc437d589e0, 0xc437d58a20, 0x0, 0x0, 0x102f900, 0xc420162ea0)\n\t/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d\ngithub.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middlewar