dump

=== received SIGQUIT ===
*** goroutine dump...
goroutine 1 [running]:
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/signals.SignalHandlerLoop(0x7fad603dc168, 0xc4200106e0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/signals/signals.go:37 +0x35c
github.com/weaveworks/cortex/server.(*Server).Run(0xc4202c2690)
	/go/src/github.com/weaveworks/cortex/server/server.go:110 +0x474
main.main()
	/go/src/github.com/weaveworks/cortex/cmd/ingester/main.go:52 +0x473

goroutine 17 [syscall, 13 minutes, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 57 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016bb50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016bb40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bb00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x3)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 56 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016bad0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016bac0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016ba80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 21 [syscall]:
os/signal.signal_recv(0x1036040)
	/usr/local/go/src/runtime/sigqueue.go:116 +0x157
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 23 [select]:
net/http.(*persistConn).roundTrip(0xc438274a00, 0xc43e614ca0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc420286000, 0xc447466d20, 0xc420286000, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc447466d20, 0x1030bc0, 0xc420286000, 0x0, 0x0, 0x0, 0x8, 0xc43d2ffa40, 0xc438377e40)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0xc4201b1590, 0xc447466d20, 0x0, 0x0, 0x0, 0xc438377e40, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0xc4201b1590, 0xc447466d20, 0xb47ac8, 0x3, 0x588a5a01, 0x106792f8)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Do(0xc4201b1590, 0xc447466d20, 0x1050960, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:184 +0x1ea
github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api.(*Client).doRequest(0xc4201865a0, 0xc4395d7900, 0xc43d2ffd30, 0xc43e614c20, 0x16, 0xc4395d7900)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api/api.go:475 +0x8e
github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api.(*KV).getInternal(0xc42017a1c8, 0xc42019a730, 0xf, 0x0, 0xc4395d78b0, 0xc420180000, 0xc43d2ffe00, 0x410558, 0x50)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api/kv.go:141 +0x233
github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api.(*KV).Get(0xc42017a1c8, 0xc42019a730, 0xf, 0xc4395d78b0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api/kv.go:75 +0x8c
github.com/weaveworks/cortex/ring.(*consulClient).WatchKey(0xc4201792e0, 0xc42019a730, 0xf, 0xc420186600, 0xc42019a720)
	/go/src/github.com/weaveworks/cortex/ring/consul_client.go:292 +0x117
github.com/weaveworks/cortex/ring.(*prefixedConsulClient).WatchKey(0xc420179300, 0xaf3adc, 0x4, 0xc420186600, 0xc42019a720)
	/go/src/github.com/weaveworks/cortex/ring/consul_client.go:353 +0xa6
github.com/weaveworks/cortex/ring.(*Ring).loop(0xc4201866c0)
	/go/src/github.com/weaveworks/cortex/ring/ring.go:119 +0xbe
created by github.com/weaveworks/cortex/ring.New
	/go/src/github.com/weaveworks/cortex/ring/ring.go:96 +0x38e

goroutine 24 [select]:
github.com/weaveworks/cortex/ring.(*IngesterRegistration).heartbeat(0xc4202444d0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ring/ingester_lifecycle.go:224 +0x3bc
github.com/weaveworks/cortex/ring.(*IngesterRegistration).loop(0xc4202444d0)
	/go/src/github.com/weaveworks/cortex/ring/ingester_lifecycle.go:153 +0xb9
created by github.com/weaveworks/cortex/ring.RegisterIngester
	/go/src/github.com/weaveworks/cortex/ring/ingester_lifecycle.go:123 +0x31f

goroutine 55 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016ba50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016ba40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016ba00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 77 [select]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc42000c300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 1283 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204962c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204962c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d5a7cd0, 0x2, 0x2, 0xc4293e5448, 0x0, 0xc42d8d94c0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d5b53e0, 0x159dc604330, 0x159dc64d710, 0xc42d5a7cd0, 0x2, 0x2, 0xc42d5b5300, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d5b53e0, 0xc42d5b52f0, 0x0, 0xc42d5a7ca8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d5b53e0, 0xa934c0, 0xc42d5b52f0, 0x7fad6035c8e0, 0xc42d5b53e0, 0x1038500, 0xc42d5b53b0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d5b53b0, 0xa934c0, 0xc42d5b52f0, 0xc42d946d40, 0xc42d946d60, 0x7fad6035c8e0, 0xc42d5b53b0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d5b53b0, 0xa934c0, 0xc42d5b52f0, 0x7fad6035c8e0, 0xc42d5b53b0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d5b53b0, 0xa934c0, 0xc42d5b52f0, 0xc42d946d40, 0xc42d946d80, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xffffffff, 0x0, 0x19, 0xc4293e57e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xc42d946d40, 0xc42d946da0, 0xc4293e5878, 0x558012, 0x588a5743, 0x26123b93)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xed01c4e43, 0x0, 0x0, 0xcea1)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xc42d946d40, 0xc42d946dc0, 0xa91b40, 0xc42d946de0, 0xc42d946d40, 0xc42d946de0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xc420246100, 0xc42d5a7c60, 0xc4293e5a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d5b51d0, 0xa934c0, 0xc42d5b52f0, 0xc42d946d40, 0xc42d946d60, 0x4e, 0xc420314b18, 0x0, 0xc420314b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d5b51d0, 0xc42d5aa420, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d054e00, 0xc420331380, 0x10265d8, 0xc42d5b5290, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d054e00, 0xc42d5b5290)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d054e00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 1015 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc420497484)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc420497480)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0xd6f76da883a32dd9)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a3e2720, 0xc420162418, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a3cbef0, 0xc42a3e2720, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a3cbef0, 0xc423f4dbc0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a3cbef0, 0xc4240da3e0, 0x0, 0xc42a3de1d8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a3cbef0, 0xa74a80, 0xc4240da3e0, 0x7fad6035c8e0, 0xc42a3cbef0, 0x1038500, 0xc42a3cbec0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a3cbec0, 0xa74a80, 0xc4240da3e0, 0xc423f4cba0, 0xc423f4cbc0, 0x7fad6035c8e0, 0xc42a3cbec0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a3cbec0, 0xa74a80, 0xc4240da3e0, 0x7fad6035c8e0, 0xc42a3cbec0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a3cbec0, 0xa74a80, 0xc4240da3e0, 0xc423f4cba0, 0xc423f4cc00, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xffffffff, 0x0, 0x19, 0xc42910b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xc423f4cba0, 0xc423f4cc40, 0xc42910b878, 0x558012, 0x588a5741, 0x3904a859)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xed01c4e41, 0x0, 0x0, 0x1646d8)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xc423f4cba0, 0xc423f4cc80, 0xa91b40, 0xc423f4cd40, 0xc423f4cba0, 0xc423f4cd40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xc420246100, 0xc42a33c9d0, 0xc42910ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a187f50, 0xa74a80, 0xc4240da3e0, 0xc423f4cba0, 0xc423f4cbc0, 0xaa6b, 0xc420044b18, 0x0, 0xc420044b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a187f50, 0xc42a1ffaa0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204ff600, 0xc420331380, 0x10265c0, 0xc42a187f80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204ff600, 0xc42a187f80)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204ff600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 76 [IO wait]:
net.runtime_pollWait(0x7fad6039d120, 0x72, 0x4)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42018c3e0, 0x72, 0xc42004b8d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42018c3e0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42018c380, 0xc4200ee000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42017a170, 0xc4200ee000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc4201860c0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc4201860c0, 0xc42000c278, 0x9, 0x9, 0x43fe80, 0xc43d04cb60, 0xc4334ef590)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc4201860c0, 0xc42000c278, 0x9, 0x9, 0x9, 0xc42004bae8, 0x451a90, 0xc447e8e1c0)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc4201860c0, 0xc42000c278, 0x9, 0x9, 0xc43d04cb60, 0x1, 0x2)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc42000c278, 0x9, 0x9, 0x102f380, 0xc4201860c0, 0x0, 0xc400000000, 0xc42004bc80, 0x7f3105)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42000c240, 0x399b, 0x399b, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420199320, 0xc432f42c00, 0xc432f42c00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42000c300, 0xc420199680, 0xb475b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc42000c300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc42017a170, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc42017a170)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 52 [select]:
github.com/weaveworks/cortex/chunk.(*MemcacheClient).updateLoop(0xc420171630, 0xdf8475800, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/memcache_client.go:78 +0x1fe
created by github.com/weaveworks/cortex/chunk.NewMemcacheClient
	/go/src/github.com/weaveworks/cortex/chunk/memcache_client.go:63 +0x24a

goroutine 54 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016b9d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016b9c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016b980, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 10433 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4384c1da0, 0xc4384c1e00, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4384c1da0, 0xc4488dea20, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4384c1da0, 0xc445991f80, 0x0, 0xc4391454a0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4384c1da0, 0xa74a80, 0xc445991f80, 0x7fad6035c8e0, 0xc4384c1da0, 0x1038500, 0xc4384c1d40)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4384c1d40, 0xa74a80, 0xc445991f80, 0xc43b967700, 0xc43b967820, 0x7fad6035c8e0, 0xc4384c1d40, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4384c1d40, 0xa74a80, 0xc445991f80, 0x7fad6035c8e0, 0xc4384c1d40, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4384c1d40, 0xa74a80, 0xc445991f80, 0xc43b967700, 0xc43b9679a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0xffffffff, 0x0, 0x19, 0xc432f7b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0xc43b967700, 0xc43b967b20, 0xc432f7b878, 0x558012, 0x588a58e2, 0xfb84a51)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0x41167e, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0xc43b967700, 0xc43b967c80, 0xa91b40, 0xc43b967dc0, 0xc43b967700, 0xc43b967dc0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0xc420246100, 0xc44538ace0, 0xc432f7ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44abb34d0, 0xa74a80, 0xc445991f80, 0xc43b967700, 0xc43b967820, 0x61d2, 0xc440d67b18, 0x0, 0xc440d67b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44abb34d0, 0xc44ac1ed80, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ac00, 0xc420331380, 0x10265c0, 0xc44ac279e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ac00, 0xc44ac279e0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ac00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 58 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016bbd0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016bbc0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bb80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x4)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 59 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016bc50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016bc40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bc00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x5)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 60 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016bcd0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016bcc0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bc80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x6)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 61 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016bd50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016bd40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bd00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x7)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 62 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016bdd0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016bdc0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bd80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x8)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 63 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016be50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016be40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016be00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x9)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 64 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016bed0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016bec0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016be80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xa)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 65 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016bf50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016bf40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bf00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xb)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 114 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42016bfd0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42016bfc0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42016bf80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xc)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 115 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c050, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c040)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c000, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xd)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 116 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c0d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c0c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c080, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xe)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 117 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c150, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c140)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c100, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0xf)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 118 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c1d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c1c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c180, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x10)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 119 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c250, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c240)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c200, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x11)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 120 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c2d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c2c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c280, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x12)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 121 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c350, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c340)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c300, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x13)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 122 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c3d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c3c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c380, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x14)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 123 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c450, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c440)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c400, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x15)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 124 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c4d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c4c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c480, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x16)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 125 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c550, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c540)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c500, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x17)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 126 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c5d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c5c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c580, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x18)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 127 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c650, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c640)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c600, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x19)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 128 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c6d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c6c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c680, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1a)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 129 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c750, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c740)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c700, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1b)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 130 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c7d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c7c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c780, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1c)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 131 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c850, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c840)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c800, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1d)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 132 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c8d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c8c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c880, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1e)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 133 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c950, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c940)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c900, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x1f)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 134 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034c9d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034c9c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034c980, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x20)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 135 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034ca50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034ca40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034ca00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x21)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 136 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034cad0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034cac0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034ca80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x22)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 137 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034cb50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034cb40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cb00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x23)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 138 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034cbd0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034cbc0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cb80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x24)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 139 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034cc50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034cc40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cc00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x25)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 140 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034ccd0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034ccc0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cc80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x26)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 141 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034cd50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034cd40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cd00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x27)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 142 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034cdd0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034cdc0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cd80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x28)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 143 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034ce50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034ce40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034ce00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x29)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 144 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034ced0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034cec0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034ce80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2a)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 145 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034cf50, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034cf40)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cf00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2b)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 146 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034cfd0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034cfc0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034cf80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2c)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 147 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034d050, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034d040)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034d000, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2d)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 148 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034d0d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034d0c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034d080, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2e)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 149 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034d150, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034d140)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034d100, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x2f)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 150 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034d1d0, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034d1c0)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034d180, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x30)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 151 [semacquire, 13 minutes]:
sync.runtime_notifyListWait(0xc42034d250, 0x0)
	/usr/local/go/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42034d240)
	/usr/local/go/src/sync/cond.go:57 +0x80
github.com/weaveworks/cortex/util.(*PriorityQueue).Dequeue(0xc42034d200, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/util/priority_queue.go:96 +0x94
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc4201e3040, 0x31)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:466 +0x89
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x9b9

goroutine 152 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3080)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).Lock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:91 +0x98
github.com/weaveworks/cortex/ingester.(*userStates).cp(0xc4201e3078, 0xc420316f8e)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:39 +0x42
github.com/weaveworks/cortex/ingester.(*Ingester).sweepUsers(0xc4201e3040, 0xc4201e4f00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:404 +0x59
github.com/weaveworks/cortex/ingester.(*Ingester).loop(0xc4201e3040)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:389 +0x134
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:187 +0xa17

goroutine 153 [IO wait]:
net.runtime_pollWait(0x7fad6039d1e0, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42010a7d0, 0x72, 0xc42004ad28, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42010a7d0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc42010a770, 0x0, 0x10312c0, 0xc42a914800)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc420038210, 0x29e8d60800, 0x0, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).AcceptTCP(0xc420038210, 0xc42004ae50, 0xc42004ae58, 0xc42004ae48)
	/usr/local/go/src/net/tcpsock.go:209 +0x49
net/http.tcpKeepAliveListener.Accept(0xc420038210, 0xb47918, 0xc422c48100, 0x1038500, 0xc4201816b0)
	/usr/local/go/src/net/http/server.go:2608 +0x2f
net/http.(*Server).Serve(0xc42007ae80, 0x1037e80, 0xc420038210, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
net/http.(*Server).ListenAndServe(0xc42007ae80, 0xc42007ae80, 0x0)
	/usr/local/go/src/net/http/server.go:2219 +0xb4
net/http.ListenAndServe(0xc4203401c8, 0x3, 0x1032a80, 0xc4203315c0, 0x3, 0x10)
	/usr/local/go/src/net/http/server.go:2351 +0xa0
created by github.com/weaveworks/cortex/server.(*Server).Run
	/go/src/github.com/weaveworks/cortex/server/server.go:100 +0x29f

goroutine 154 [IO wait, 13 minutes]:
net.runtime_pollWait(0x7fad6039d060, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420242f40, 0x72, 0xc42023fd70, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420242f40, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420242ee0, 0x0, 0x10312c0, 0xc43e0cd620)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42016e1b8, 0x43356e, 0xc42023fe20, 0x63e1dc)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc42016e1b8, 0xb47580, 0xc4202460c0, 0x103ae60, 0xc43a4a07d8)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve(0xc4202460c0, 0x1037580, 0xc42016e1b8, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:368 +0x187
created by github.com/weaveworks/cortex/server.(*Server).Run
	/go/src/github.com/weaveworks/cortex/server/server.go:108 +0x419

goroutine 155 [select, 13 minutes, locked to thread]:
runtime.gopark(0xb47f10, 0x0, 0xaf59e3, 0x6, 0x18, 0x2)
	/usr/local/go/src/runtime/proc.go:259 +0x13a
runtime.selectgoImpl(0xc42036af30, 0x0, 0x18)
	/usr/local/go/src/runtime/select.go:423 +0x11d9
runtime.selectgo(0xc42036af30)
	/usr/local/go/src/runtime/select.go:238 +0x1c
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal1_unix.go:304 +0x2f3
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 8355 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42c7ac4e0, 0xc42c7ac510, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42c7ac4e0, 0xc43a3ea9e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42c7ac4e0, 0xc43ddc5ba0, 0x0, 0xc42c7ab458, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42c7ac4e0, 0xa74a80, 0xc43ddc5ba0, 0x7fad6035c8e0, 0xc42c7ac4e0, 0x1038500, 0xc42c7ac4b0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c7ac4b0, 0xa74a80, 0xc43ddc5ba0, 0xc43a3ea8e0, 0xc43a3ea900, 0x7fad6035c8e0, 0xc42c7ac4b0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c7ac4b0, 0xa74a80, 0xc43ddc5ba0, 0x7fad6035c8e0, 0xc42c7ac4b0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c7ac4b0, 0xa74a80, 0xc43ddc5ba0, 0xc43a3ea8e0, 0xc43a3ea920, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xffffffff, 0x0, 0x19, 0xc43a6677e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xc43a3ea8e0, 0xc43a3ea940, 0xc43a667878, 0x558012, 0x588a579b, 0x39a1dd86)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xed01c4e9b, 0x0, 0x0, 0xad19c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xc43a3ea8e0, 0xc43a3ea960, 0xa91b40, 0xc43a3ea980, 0xc43a3ea8e0, 0xc43a3ea980)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xc420246100, 0xc42cb083a0, 0xc43a667a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42cbe5cb0, 0xa74a80, 0xc43ddc5ba0, 0xc43a3ea8e0, 0xc43a3ea900, 0x8755, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42cbe5cb0, 0xc42cbf63c0, 0xc420179d60, 0x0, 0x0, 0x637573070a0f0a63, 0x7274041273736563)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad700, 0xc420331380, 0x10265c0, 0xc42cbe5ce0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad700, 0xc42cbe5ce0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad700)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 12099 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc440ce2f00, 0xc440ce2f30, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc440ce2f00, 0xc4437ad520, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc440ce2f00, 0xc448583840, 0x0, 0xc42b08ced8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc440ce2f00, 0xa74a80, 0xc448583840, 0x7fad6035c8e0, 0xc440ce2f00, 0x1038500, 0xc440ce2ed0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc440ce2ed0, 0xa74a80, 0xc448583840, 0xc4437ad3e0, 0xc4437ad400, 0x7fad6035c8e0, 0xc440ce2ed0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440ce2ed0, 0xa74a80, 0xc448583840, 0x7fad6035c8e0, 0xc440ce2ed0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc440ce2ed0, 0xa74a80, 0xc448583840, 0xc4437ad3e0, 0xc4437ad440, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0xffffffff, 0x0, 0x19, 0xc4371bb7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0xc4437ad3e0, 0xc4437ad460, 0xc4371bb878, 0x558012, 0x588a59f0, 0x102a6306)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0x41167e, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0xc4437ad3e0, 0xc4437ad480, 0xa91b40, 0xc4437ad4a0, 0xc4437ad3e0, 0xc4437ad4a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0xc420246100, 0xc42c15d720, 0xc4371bba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44809f890, 0xa74a80, 0xc448583840, 0xc4437ad3e0, 0xc4437ad400, 0x7f2c, 0xc420313b18, 0x0, 0xc420313b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44809f890, 0xc43dc3de60, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc43859e400, 0xc420331380, 0x10265c0, 0xc44809f8c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc43859e400, 0xc44809f8c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc43859e400)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8919 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43756d4d0, 0xc43756d500, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43756d4d0, 0xc442132180, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43756d4d0, 0xc44623ef60, 0x0, 0xc42b6f1f08, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43756d4d0, 0xa74a80, 0xc44623ef60, 0x7fad6035c8e0, 0xc43756d4d0, 0x1038500, 0xc43756d4a0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43756d4a0, 0xa74a80, 0xc44623ef60, 0xc442132080, 0xc4421320a0, 0x7fad6035c8e0, 0xc43756d4a0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43756d4a0, 0xa74a80, 0xc44623ef60, 0x7fad6035c8e0, 0xc43756d4a0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43756d4a0, 0xa74a80, 0xc44623ef60, 0xc442132080, 0xc4421320c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xffffffff, 0x0, 0x19, 0xc44a87b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xc442132080, 0xc4421320e0, 0xc44a87b878, 0x558012, 0x588a57f2, 0xf0c3774)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xed01c4ef2, 0x0, 0x0, 0x3fcd45)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xc442132080, 0xc442132100, 0xa91b40, 0xc442132120, 0xc442132080, 0xc442132120)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xc420246100, 0xc4278f6360, 0xc44a87ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442347680, 0xa74a80, 0xc44623ef60, 0xc442132080, 0xc4421320a0, 0x4f3f, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442347680, 0xc42a5f5380, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5986c5)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ad00, 0xc420331380, 0x10265c0, 0xc4423476b0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ad00, 0xc4423476b0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439a5ad00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8736 [chan receive, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc43abf3ae0, 0xc44776e870)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc43abf3ae0, 0xc44776e870)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc43abf3ae0, 0xc44776e870)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc4324df4c0, 0xc44776e870)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc4324df4c0, 0xc44776e870)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc4324df4c0, 0xc44776e870)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc4324df4a0, 0xc44776e690)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc4324df4a0, 0xc44776e690)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc4324df480, 0xc44776e690)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc4324df480, 0xc44776e690)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4252bde10, 0xc44776e690)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4252bde10, 0xc44776e5a0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4252bde10, 0xc44776e5a0)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc43aeff500, 0x1038440, 0xc423686fc0)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 5313 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048c504)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048c500)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x2dce913412292e09)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc446b45da0, 0xc422334c2d, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc446b458c0, 0xc446b45da0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc446b458c0, 0xc42b71b300, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc446b458c0, 0xc42ba151c0, 0x0, 0xc446a05338, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc446b458c0, 0xa74a80, 0xc42ba151c0, 0x7fad6035c8e0, 0xc446b458c0, 0x1038500, 0xc446b45890)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc446b45890, 0xa74a80, 0xc42ba151c0, 0xc42b71ac00, 0xc42b71ac20, 0x7fad6035c8e0, 0xc446b45890, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc446b45890, 0xa74a80, 0xc42ba151c0, 0x7fad6035c8e0, 0xc446b45890, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc446b45890, 0xa74a80, 0xc42ba151c0, 0xc42b71ac00, 0xc42b71ac60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xffffffff, 0x0, 0x19, 0xc42d70d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xc42b71ac00, 0xc42b71ac80, 0xc42d70d878, 0x558012, 0x588a575f, 0x344a53de)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xed01c4e5f, 0x0, 0x0, 0x173b0c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xc42b71ac00, 0xc42b71aca0, 0xa91b40, 0xc42b71ace0, 0xc42b71ac00, 0xc42b71ace0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xc420246100, 0xc4494205e0, 0xc42d70da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc447b11e00, 0xa74a80, 0xc42ba151c0, 0xc42b71ac00, 0xc42b71ac20, 0x8af4, 0xc440d67b18, 0x0, 0xc440d67b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc447b11e00, 0xc43c5cbb00, 0xc420179d60, 0x0, 0x0, 0xc42ad71fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cbe300, 0xc420331380, 0x10265c0, 0xc447b11e90, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cbe300, 0xc447b11e90)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cbe300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8321 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a56bd40, 0xc42a56bd70, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a56bd40, 0xc437d05d80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a56bd40, 0xc437d59b60, 0x0, 0xc42a6869d8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a56bd40, 0xa74a80, 0xc437d59b60, 0x7fad6035c8e0, 0xc42a56bd40, 0x1038500, 0xc42a56bd10)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a56bd10, 0xa74a80, 0xc437d59b60, 0xc437d05c40, 0xc437d05c60, 0x7fad6035c8e0, 0xc42a56bd10, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a56bd10, 0xa74a80, 0xc437d59b60, 0x7fad6035c8e0, 0xc42a56bd10, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a56bd10, 0xa74a80, 0xc437d59b60, 0xc437d05c40, 0xc437d05c80, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xffffffff, 0x0, 0x19, 0xc4464b77e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xc437d05c40, 0xc437d05ca0, 0xc4464b7878, 0x558012, 0x588a5798, 0xdb38f7a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xed01c4e98, 0x0, 0x0, 0x648265)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xc437d05c40, 0xc437d05d00, 0xa91b40, 0xc437d05d20, 0xc437d05c40, 0xc437d05d20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xc420246100, 0xc42b9ddd60, 0xc4464b7a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42f4074a0, 0xa74a80, 0xc437d59b60, 0xc437d05c40, 0xc437d05c60, 0x5d64, 0xc420046b18, 0x0, 0xc420046b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42f4074a0, 0xc4296fa3c0, 0xc420179d60, 0x0, 0x0, 0xc4202e17a8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67f00, 0xc420331380, 0x10265c0, 0xc42f4074d0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67f00, 0xc42f4074d0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67f00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8322 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4288a6d20, 0xc4288a6d50, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4288a6d20, 0xc437da04c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4288a6d20, 0xc437e1f420, 0x0, 0xc42c8d46d0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4288a6d20, 0xa74a80, 0xc437e1f420, 0x7fad6035c8e0, 0xc4288a6d20, 0x1038500, 0xc4288a6cf0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4288a6cf0, 0xa74a80, 0xc437e1f420, 0xc437da03a0, 0xc437da03c0, 0x7fad6035c8e0, 0xc4288a6cf0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4288a6cf0, 0xa74a80, 0xc437e1f420, 0x7fad6035c8e0, 0xc4288a6cf0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4288a6cf0, 0xa74a80, 0xc437e1f420, 0xc437da03a0, 0xc437da03e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xffffffff, 0x0, 0x19, 0xc43b4f17e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xc437da03a0, 0xc437da0400, 0xc43b4f1878, 0x558012, 0x588a5798, 0xda24519)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xed01c4e98, 0x0, 0x0, 0x1698ea)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xc437da03a0, 0xc437da0420, 0xa91b40, 0xc437da0440, 0xc437da03a0, 0xc437da0440)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xc420246100, 0xc426fafea0, 0xc43b4f1a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42f407740, 0xa74a80, 0xc437e1f420, 0xc437da03a0, 0xc437da03c0, 0x4351, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42f407740, 0xc44363f800, 0xc420179d60, 0x0, 0x0, 0xc4202e17a8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c2d00, 0xc420331380, 0x10265c0, 0xc42f407770, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c2d00, 0xc42f407770)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c2d00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8734 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc448c247b0, 0xc448c247e0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc448c247b0, 0xc43287f920, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc448c247b0, 0xc43333ff80, 0x0, 0xc448761bf0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc448c247b0, 0xa74a80, 0xc43333ff80, 0x7fad6035c8e0, 0xc448c247b0, 0x1038500, 0xc448c24780)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc448c24780, 0xa74a80, 0xc43333ff80, 0xc43287f800, 0xc43287f820, 0x7fad6035c8e0, 0xc448c24780, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc448c24780, 0xa74a80, 0xc43333ff80, 0x7fad6035c8e0, 0xc448c24780, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc448c24780, 0xa74a80, 0xc43333ff80, 0xc43287f800, 0xc43287f840, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xffffffff, 0x0, 0x19, 0xc42f3897e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xc43287f800, 0xc43287f860, 0xc42f389878, 0x558012, 0x588a57d4, 0xe8eaebe)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xed01c4ed4, 0x0, 0x0, 0x2ec2bf)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xc43287f800, 0xc43287f880, 0xa91b40, 0xc43287f8a0, 0xc43287f800, 0xc43287f8a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xc420246100, 0xc438653d90, 0xc42f389a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43fc60c90, 0xa74a80, 0xc43333ff80, 0xc43287f800, 0xc43287f820, 0x5241, 0xc420e1ab18, 0x0, 0xc420e1ab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43fc60c90, 0xc44993a0c0, 0xc420179d60, 0x0, 0x0, 0xc43bd367a8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421000d00, 0xc420331380, 0x10265c0, 0xc43fc60cc0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421000d00, 0xc43fc60cc0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421000d00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 1119 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204962c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204962c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42b839760, 0x2, 0x2, 0xc4260f7448, 0x0, 0xc42b842380)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42b850a80, 0x159dc604330, 0x159dc64d710, 0xc42b839760, 0x2, 0x2, 0xc42b850a00, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42b850a80, 0xc42b850990, 0x0, 0xc42b839738, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42b850a80, 0xa934c0, 0xc42b850990, 0x7fad6035c8e0, 0xc42b850a80, 0x1038500, 0xc42b850a50)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42b850a50, 0xa934c0, 0xc42b850990, 0xc42b854740, 0xc42b854760, 0x7fad6035c8e0, 0xc42b850a50, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b850a50, 0xa934c0, 0xc42b850990, 0x7fad6035c8e0, 0xc42b850a50, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42b850a50, 0xa934c0, 0xc42b850990, 0xc42b854740, 0xc42b854780, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xffffffff, 0x0, 0x19, 0xc4260f77e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xc42b854740, 0xc42b8547a0, 0xc4260f7878, 0x558012, 0x588a5742, 0x213cc5e5)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xc4260f7900, 0x0, 0x0, 0x3f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xc42b854740, 0xc42b8547c0, 0xa91b40, 0xc42b8547e0, 0xc42b854740, 0xc42b8547e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xc420246100, 0xc42b8396f0, 0xc4260f7a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42b850870, 0xa934c0, 0xc42b850990, 0xc42b854740, 0xc42b854760, 0x4e, 0xc420047b18, 0x0, 0xc420047b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42b850870, 0xc42b5f0660, 0xc420179d60, 0x0, 0x0, 0x2d65646f6e2d6d6f, 0x726574726f707865)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429907200, 0xc420331380, 0x10265d8, 0xc42b850930, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429907200, 0xc42b850930)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429907200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 5574 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7fad6039cd60, 0x72, 0x3)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc43d2bf5d0, 0x72, 0xc440d668d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc43d2bf5d0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc43d2bf570, 0xc43c8de000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4464e30c0, 0xc43c8de000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc445d82fc0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc445d82fc0, 0xc42000d6b8, 0x9, 0x9, 0x43fe80, 0xc4351c4240, 0x0)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc445d82fc0, 0xc42000d6b8, 0x9, 0x9, 0x9, 0xc440d66ae8, 0x451a90, 0xc42b28bd00)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc445d82fc0, 0xc42000d6b8, 0x9, 0x9, 0xc4351c4240, 0x0, 0x1)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc42000d6b8, 0x9, 0x9, 0x102f380, 0xc445d82fc0, 0x0, 0xc400000000, 0xc440d66c80, 0x7f3105)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42000d680, 0xf, 0xf, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4408cc8d0, 0xc42f4308a0, 0xc42f4308a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42000d740, 0xc4408cc960, 0xb475b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc42000d740)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc4464e30c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc4464e30c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 870 [IO wait, 13 minutes]:
net.runtime_pollWait(0x7fad6039cee0, 0x72, 0x9)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc427bb3b80, 0x72, 0xc4203128d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc427bb3b80, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc427bb3b20, 0xc4293a6000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc428d0b6f8, 0xc4293a6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc4287d9d40)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc4287d9d40, 0xc426048278, 0x9, 0x9, 0x43fe80, 0xc42de31f20, 0x0)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc4287d9d40, 0xc426048278, 0x9, 0x9, 0x9, 0xc420312ae8, 0x451a90, 0xc421266a80)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc4287d9d40, 0xc426048278, 0x9, 0x9, 0xc42de31f20, 0x0, 0x1)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc426048278, 0x9, 0x9, 0x102f380, 0xc4287d9d40, 0x0, 0xc400000000, 0xc420312c80, 0x7f3105)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc426048240, 0x5a, 0x5a, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc429291620, 0xc42de39c80, 0xc42de39c80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc426048300, 0xc4292916b0, 0xb475b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc426048300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc428d0b6f8, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc428d0b6f8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 11264 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42516a930, 0xc42516a960, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42516a930, 0xc43c693ee0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42516a930, 0xc4438a9a60, 0x0, 0xc4320da8c0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42516a930, 0xa74a80, 0xc4438a9a60, 0x7fad6035c8e0, 0xc42516a930, 0x1038500, 0xc42516a900)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42516a900, 0xa74a80, 0xc4438a9a60, 0xc43c693de0, 0xc43c693e00, 0x7fad6035c8e0, 0xc42516a900, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42516a900, 0xa74a80, 0xc4438a9a60, 0x7fad6035c8e0, 0xc42516a900, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42516a900, 0xa74a80, 0xc4438a9a60, 0xc43c693de0, 0xc43c693e20, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xffffffff, 0x0, 0x19, 0xc43618b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xc43c693de0, 0xc43c693e40, 0xc43618b878, 0x558012, 0x588a597b, 0x39877a42)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xed01c507b, 0x0, 0x0, 0xe3173)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xc43c693de0, 0xc43c693e60, 0xa91b40, 0xc43c693e80, 0xc43c693de0, 0xc43c693e80)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xc420246100, 0xc4322de280, 0xc43618ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4243633b0, 0xa74a80, 0xc4438a9a60, 0xc43c693de0, 0xc43c693e00, 0x6ec4, 0xc440d67b18, 0x0, 0xc440d67b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4243633b0, 0xc44346cf60, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f62ebe6)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc43859ec00, 0xc420331380, 0x10265c0, 0xc4243633e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc43859ec00, 0xc4243633e0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc43859ec00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8981 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4424c1f50, 0xc4424c1f80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4424c1f50, 0xc44a255540, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4424c1f50, 0xc43adc00e0, 0x0, 0xc43e30b9b8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4424c1f50, 0xa74a80, 0xc43adc00e0, 0x7fad6035c8e0, 0xc4424c1f50, 0x1038500, 0xc4424c1f20)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4424c1f20, 0xa74a80, 0xc43adc00e0, 0xc44a255440, 0xc44a255460, 0x7fad6035c8e0, 0xc4424c1f20, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4424c1f20, 0xa74a80, 0xc43adc00e0, 0x7fad6035c8e0, 0xc4424c1f20, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4424c1f20, 0xa74a80, 0xc43adc00e0, 0xc44a255440, 0xc44a255480, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xffffffff, 0x0, 0x19, 0xc44a6737e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xc44a255440, 0xc44a2554a0, 0xc44a673878, 0x558012, 0x588a5810, 0xd29b3f4)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xed01c4f10, 0x0, 0x0, 0x739753)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xc44a255440, 0xc44a2554c0, 0xa91b40, 0xc44a2554e0, 0xc44a255440, 0xc44a2554e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xc420246100, 0xc441187f00, 0xc44a673a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43f9e3f80, 0xa74a80, 0xc43adc00e0, 0xc44a255440, 0xc44a255460, 0x7abb, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43f9e3f80, 0xc4408703c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ece23fd)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0700, 0xc420331380, 0x10265c0, 0xc43f9e3fb0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0700, 0xc43f9e3fb0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0700)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10206 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4357a62d0, 0xc4357a6300, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4357a62d0, 0xc445cf3f00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4357a62d0, 0xc448b04c20, 0x0, 0xc4370bdd10, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4357a62d0, 0xa74a80, 0xc448b04c20, 0x7fad6035c8e0, 0xc4357a62d0, 0x1038500, 0xc4357a62a0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4357a62a0, 0xa74a80, 0xc448b04c20, 0xc445cf32c0, 0xc445cf32e0, 0x7fad6035c8e0, 0xc4357a62a0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4357a62a0, 0xa74a80, 0xc448b04c20, 0x7fad6035c8e0, 0xc4357a62a0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4357a62a0, 0xa74a80, 0xc448b04c20, 0xc445cf32c0, 0xc445cf3320, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xffffffff, 0x0, 0x19, 0xc44226f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xc445cf32c0, 0xc445cf3340, 0xc44226f878, 0x558012, 0x588a58c7, 0x3a488275)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xc44226f900, 0x0, 0x0, 0xf4)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xc445cf32c0, 0xc445cf3e80, 0xa91b40, 0xc445cf3ea0, 0xc445cf32c0, 0xc445cf3ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xc420246100, 0xc43714e850, 0xc44226fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43580eae0, 0xa74a80, 0xc448b04c20, 0xc445cf32c0, 0xc445cf32e0, 0xb442, 0xc420319b18, 0x0, 0xc420319b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43580eae0, 0xc431058ae0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ec6ce01)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc434d94a00, 0xc420331380, 0x10265c0, 0xc43580eb70, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc434d94a00, 0xc43580eb70)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc434d94a00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 977 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204895c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204895c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x4fbdccb03de0ad07)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a2e7920, 0xc422334c2d, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a2e6b10, 0xc42a2e7920, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a2e6b10, 0xc424223f80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a2e6b10, 0xc4244736e0, 0x0, 0xc42a2e1e68, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a2e6b10, 0xa74a80, 0xc4244736e0, 0x7fad6035c8e0, 0xc42a2e6b10, 0x1038500, 0xc42a2e6ae0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a2e6ae0, 0xa74a80, 0xc4244736e0, 0xc424233800, 0xc424233820, 0x7fad6035c8e0, 0xc42a2e6ae0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a2e6ae0, 0xa74a80, 0xc4244736e0, 0x7fad6035c8e0, 0xc42a2e6ae0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a2e6ae0, 0xa74a80, 0xc4244736e0, 0xc424233800, 0xc424233860, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xffffffff, 0x0, 0x19, 0xc4294757e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xc424233800, 0xc424233880, 0xc429475878, 0x558012, 0x588a5741, 0x3761df55)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xed01c4e41, 0x0, 0x0, 0x517a64)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xc424233800, 0xc4242338c0, 0xa91b40, 0xc4242338e0, 0xc424233800, 0xc4242338e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xc420246100, 0xc42a1b7f50, 0xc429475a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc425e39530, 0xa74a80, 0xc4244736e0, 0xc424233800, 0xc424233820, 0xc28a, 0xc42023bb18, 0x0, 0xc42023bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc425e39530, 0xc429d97920, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cadc00, 0xc420331380, 0x10265c0, 0xc42a2016b0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cadc00, 0xc42a2016b0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cadc00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8983 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43b9a7200, 0xc43b9a7230, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43b9a7200, 0xc4496c0920, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43b9a7200, 0xc43dd60000, 0x0, 0xc4425b0870, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43b9a7200, 0xa74a80, 0xc43dd60000, 0x7fad6035c8e0, 0xc43b9a7200, 0x1038500, 0xc43b9a71d0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43b9a71d0, 0xa74a80, 0xc43dd60000, 0xc4496c0820, 0xc4496c0840, 0x7fad6035c8e0, 0xc43b9a71d0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b9a71d0, 0xa74a80, 0xc43dd60000, 0x7fad6035c8e0, 0xc43b9a71d0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43b9a71d0, 0xa74a80, 0xc43dd60000, 0xc4496c0820, 0xc4496c0860, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xffffffff, 0x0, 0x19, 0xc44a6757e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xc4496c0820, 0xc4496c0880, 0xc44a675878, 0x558012, 0x588a5810, 0xd39d62f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xed01c4f10, 0x0, 0x0, 0x3b9748)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xc4496c0820, 0xc4496c08a0, 0xa91b40, 0xc4496c08c0, 0xc4496c0820, 0xc4496c08c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xc420246100, 0xc43e30ba10, 0xc44a675a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43caf4390, 0xa74a80, 0xc43dd60000, 0xc4496c0820, 0xc4496c0840, 0x8cd7, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43caf4390, 0xc4408709c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f400720)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0b00, 0xc420331380, 0x10265c0, 0xc43caf4420, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0b00, 0xc43caf4420)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0b00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 330 [select]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc420246600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 8920 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc449d8f350, 0xc449d8f380, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc449d8f350, 0xc44a4d00e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc449d8f350, 0xc44a505c40, 0x0, 0xc44479a7e0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc449d8f350, 0xa74a80, 0xc44a505c40, 0x7fad6035c8e0, 0xc449d8f350, 0x1038500, 0xc449d8f320)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc449d8f320, 0xa74a80, 0xc44a505c40, 0xc44a4e3fe0, 0xc44a4d0000, 0x7fad6035c8e0, 0xc449d8f320, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449d8f320, 0xa74a80, 0xc44a505c40, 0x7fad6035c8e0, 0xc449d8f320, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc449d8f320, 0xa74a80, 0xc44a505c40, 0xc44a4e3fe0, 0xc44a4d0020, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xffffffff, 0x0, 0x19, 0xc43d7c57e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xc44a4e3fe0, 0xc44a4d0040, 0xc43d7c5878, 0x558012, 0x588a57f5, 0x397ee00c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xed01c4ef5, 0x0, 0x0, 0x223e3)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xc44a4e3fe0, 0xc44a4d0060, 0xa91b40, 0xc44a4d0080, 0xc44a4e3fe0, 0xc44a4d0080)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xc420246100, 0xc445048110, 0xc43d7c5a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc449d8e9c0, 0xa74a80, 0xc44a505c40, 0xc44a4e3fe0, 0xc44a4d0000, 0x16c8, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc449d8e9c0, 0xc42c213f80, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5986c5)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b200, 0xc420331380, 0x10265c0, 0xc449d8ea80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b200, 0xc449d8ea80)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9665 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43956cf30, 0xc43956cf60, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43956cf30, 0xc449c42860, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43956cf30, 0xc449e2a000, 0x0, 0xc43c1e6218, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43956cf30, 0xa74a80, 0xc449e2a000, 0x7fad6035c8e0, 0xc43956cf30, 0x1038500, 0xc43956cf00)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43956cf00, 0xa74a80, 0xc449e2a000, 0xc449c42760, 0xc449c42780, 0x7fad6035c8e0, 0xc43956cf00, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43956cf00, 0xa74a80, 0xc449e2a000, 0x7fad6035c8e0, 0xc43956cf00, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43956cf00, 0xa74a80, 0xc449e2a000, 0xc449c42760, 0xc449c427a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xffffffff, 0x0, 0x19, 0xc42eb397e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xc449c42760, 0xc449c427c0, 0xc42eb39878, 0x558012, 0x588a586a, 0x103e60fa)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xc42eb39900, 0x0, 0x0, 0x3f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xc449c42760, 0xc449c427e0, 0xa91b40, 0xc449c42800, 0xc449c42760, 0xc449c42800)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xc420246100, 0xc4446e8400, 0xc42eb39a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43dff13e0, 0xa74a80, 0xc449e2a000, 0xc449c42760, 0xc449c42780, 0x75cf, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43dff13e0, 0xc441eb6000, 0xc420179d60, 0x0, 0x0, 0x7fad625dc000, 0xc41eac8542)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87b00, 0xc420331380, 0x10265c0, 0xc43dff1470, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87b00, 0xc43dff1470)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87b00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8735 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc443f03b60, 0xc443f03b90, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc443f03b60, 0xc4325bf180, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc443f03b60, 0xc43281e2e0, 0x0, 0xc443ff55d8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc443f03b60, 0xa74a80, 0xc43281e2e0, 0x7fad6035c8e0, 0xc443f03b60, 0x1038500, 0xc443f03b30)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc443f03b30, 0xa74a80, 0xc43281e2e0, 0xc4325bf000, 0xc4325bf020, 0x7fad6035c8e0, 0xc443f03b30, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc443f03b30, 0xa74a80, 0xc43281e2e0, 0x7fad6035c8e0, 0xc443f03b30, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc443f03b30, 0xa74a80, 0xc43281e2e0, 0xc4325bf000, 0xc4325bf040, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xffffffff, 0x0, 0x19, 0xc43e8bb7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xc4325bf000, 0xc4325bf060, 0xc43e8bb878, 0x558012, 0x588a57d7, 0x3a7e9931)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xed01c4ed7, 0x0, 0x0, 0x6463c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xc4325bf000, 0xc4325bf0a0, 0xa91b40, 0xc4325bf0e0, 0xc4325bf000, 0xc4325bf0e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xc420246100, 0xc4463ebf00, 0xc43e8bba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc443f02870, 0xa74a80, 0xc43281e2e0, 0xc4325bf000, 0xc4325bf020, 0x4254, 0xc420e1ab18, 0x0, 0xc420e1ab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc443f02870, 0xc44661b980, 0xc420179d60, 0x0, 0x0, 0x10, 0xc42ac41698)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421001000, 0xc420331380, 0x10265c0, 0xc443f02990, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421001000, 0xc443f02990)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421001000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10207 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4356a7560, 0xc4356a75f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4356a7560, 0xc443e900c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4356a7560, 0xc447578260, 0x0, 0xc4370235a0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4356a7560, 0xa74a80, 0xc447578260, 0x7fad6035c8e0, 0xc4356a7560, 0x1038500, 0xc4356a74d0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4356a74d0, 0xa74a80, 0xc447578260, 0xc44a24ffa0, 0xc44a24ffc0, 0x7fad6035c8e0, 0xc4356a74d0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4356a74d0, 0xa74a80, 0xc447578260, 0x7fad6035c8e0, 0xc4356a74d0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4356a74d0, 0xa74a80, 0xc447578260, 0xc44a24ffa0, 0xc44a24ffe0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xffffffff, 0x0, 0x19, 0xc442c1d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xc44a24ffa0, 0xc443e90000, 0xc442c1d878, 0x558012, 0x588a58c7, 0x3a6a326f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xc442c1d900, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xc44a24ffa0, 0xc443e90020, 0xa91b40, 0xc443e90040, 0xc44a24ffa0, 0xc443e90040)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xc420246100, 0xc437237400, 0xc442c1da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43575dc20, 0xa74a80, 0xc447578260, 0xc44a24ffa0, 0xc44a24ffc0, 0xb624, 0xc420319b18, 0x0, 0xc420319b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43575dc20, 0xc4320cd260, 0xc420179d60, 0x0, 0x0, 0xc43b43a600, 0xb)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434d94e00, 0xc420331380, 0x10265c0, 0xc43575dc80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434d94e00, 0xc43575dc80)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434d94e00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8319 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4248f06c0, 0xc4248f06f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4248f06c0, 0xc437e1e300, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4248f06c0, 0xc437e3fbe0, 0x0, 0xc426fafd58, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4248f06c0, 0xa74a80, 0xc437e3fbe0, 0x7fad6035c8e0, 0xc4248f06c0, 0x1038500, 0xc4248f0690)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4248f0690, 0xa74a80, 0xc437e3fbe0, 0xc437e1e180, 0xc437e1e1c0, 0x7fad6035c8e0, 0xc4248f0690, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f0690, 0xa74a80, 0xc437e3fbe0, 0x7fad6035c8e0, 0xc4248f0690, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4248f0690, 0xa74a80, 0xc437e3fbe0, 0xc437e1e180, 0xc437e1e200, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xffffffff, 0x0, 0x19, 0xc4293e17e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xc437e1e180, 0xc437e1e220, 0xc4293e1878, 0x558012, 0x588a5798, 0xd9a631c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xed01c4e98, 0x0, 0x0, 0x4ca84e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xc437e1e180, 0xc437e1e260, 0xa91b40, 0xc437e1e280, 0xc437e1e180, 0xc437e1e280)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xc420246100, 0xc43d7955b0, 0xc4293e1a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42f407080, 0xa74a80, 0xc437e3fbe0, 0xc437e1e180, 0xc437e1e1c0, 0x454f, 0xc420e1ab18, 0x0, 0xc420e1ab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42f407080, 0xc44363f560, 0xc420179d60, 0x0, 0x0, 0xc4202e17a8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67d00, 0xc420331380, 0x10265c0, 0xc42f4072c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67d00, 0xc42f4072c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc428c67d00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9507 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4388fa1e0, 0xc4388fa210, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4388fa1e0, 0xc4472e70e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4388fa1e0, 0xc4476a40a0, 0x0, 0xc4393d2f50, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4388fa1e0, 0xa74a80, 0xc4476a40a0, 0x7fad6035c8e0, 0xc4388fa1e0, 0x1038500, 0xc4388fa1b0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4388fa1b0, 0xa74a80, 0xc4476a40a0, 0xc4472e6fe0, 0xc4472e7000, 0x7fad6035c8e0, 0xc4388fa1b0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4388fa1b0, 0xa74a80, 0xc4476a40a0, 0x7fad6035c8e0, 0xc4388fa1b0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4388fa1b0, 0xa74a80, 0xc4476a40a0, 0xc4472e6fe0, 0xc4472e7020, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xffffffff, 0x0, 0x19, 0xc449dad7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xc4472e6fe0, 0xc4472e7040, 0xc449dad878, 0x558012, 0x588a584f, 0x3b937843)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xed01c4f4f, 0x0, 0x0, 0x6225e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xc4472e6fe0, 0xc4472e7060, 0xa91b40, 0xc4472e7080, 0xc4472e6fe0, 0xc4472e7080)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xc420246100, 0xc43940c4e0, 0xc449dada70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43891e450, 0xa74a80, 0xc4476a40a0, 0xc4472e6fe0, 0xc4472e7000, 0x3352, 0xc42023ab18, 0x0, 0xc42023ab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43891e450, 0xc4377bcae0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ec3b394)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064d00, 0xc420331380, 0x10265c0, 0xc43891e480, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064d00, 0xc43891e480)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064d00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 5327 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048c504)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048c500)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x4dbfada6ed50c1e2)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc444637c20, 0xc420162418, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc444637320, 0xc444637c20, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc444637320, 0xc42b39d080, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc444637320, 0xc42b5a9ba0, 0x0, 0xc443fc3f78, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc444637320, 0xa74a80, 0xc42b5a9ba0, 0x7fad6035c8e0, 0xc444637320, 0x1038500, 0xc4446372f0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4446372f0, 0xa74a80, 0xc42b5a9ba0, 0xc42b39c780, 0xc42b39c7a0, 0x7fad6035c8e0, 0xc4446372f0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4446372f0, 0xa74a80, 0xc42b5a9ba0, 0x7fad6035c8e0, 0xc4446372f0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4446372f0, 0xa74a80, 0xc42b5a9ba0, 0xc42b39c780, 0xc42b39c7c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xffffffff, 0x0, 0x19, 0xc4359957e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xc42b39c780, 0xc42b39c7e0, 0xc435995878, 0x558012, 0x588a575f, 0x39a47310)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xc435995900, 0x0, 0x0, 0x3f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xc42b39c780, 0xc42b39c800, 0xa91b40, 0xc42b39c820, 0xc42b39c780, 0xc42b39c820)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xc420246100, 0xc444f0e800, 0xc435995a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc444f08f60, 0xa74a80, 0xc42b5a9ba0, 0xc42b39c780, 0xc42b39c7a0, 0x962f, 0xc420e1ab18, 0x0, 0xc420e1ab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc444f08f60, 0xc43bba4a20, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cb4000, 0xc420331380, 0x10265c0, 0xc444f08f90, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cb4000, 0xc444f08f90)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cb4000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9882 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437ceb530, 0xc437ceb590, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437ceb530, 0xc449d7dd40, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437ceb530, 0xc4410fc520, 0x0, 0xc438c38dd8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437ceb530, 0xa74a80, 0xc4410fc520, 0x7fad6035c8e0, 0xc437ceb530, 0x1038500, 0xc437ceb4d0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc437ceb4d0, 0xa74a80, 0xc4410fc520, 0xc449d7dc20, 0xc449d7dc40, 0x7fad6035c8e0, 0xc437ceb4d0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437ceb4d0, 0xa74a80, 0xc4410fc520, 0x7fad6035c8e0, 0xc437ceb4d0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc437ceb4d0, 0xa74a80, 0xc4410fc520, 0xc449d7dc20, 0xc449d7dc60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xffffffff, 0x0, 0x19, 0xc4414597e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xc449d7dc20, 0xc449d7dc80, 0xc441459878, 0x558012, 0x588a5888, 0x104d9a02)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xed01c4f88, 0x0, 0x0, 0x8fe8c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xc449d7dc20, 0xc449d7dcc0, 0xa91b40, 0xc449d7dce0, 0xc449d7dc20, 0xc449d7dce0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xc420246100, 0xc438d1e890, 0xc441459a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc437eefd40, 0xa74a80, 0xc4410fc520, 0xc449d7dc20, 0xc449d7dc40, 0x600d, 0xc42023bb18, 0x0, 0xc42023bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc437eefd40, 0xc441196f00, 0xc420179d60, 0x0, 0x0, 0xc439252d40, 0x646e696b)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5300, 0xc420331380, 0x10265c0, 0xc437eefe60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5300, 0xc437eefe60)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9218 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a
github.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc4469ffbc0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc43f716b10, 0xc4469ffbc0, 0x10343c0, 0xc4201e3040)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326

goroutine 12081 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4496097a0, 0xc4496097d0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4496097a0, 0xc4457267a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4496097a0, 0xc445b966a0, 0x0, 0xc427f8c868, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4496097a0, 0xa74a80, 0xc445b966a0, 0x7fad6035c8e0, 0xc4496097a0, 0x1038500, 0xc449609770)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc449609770, 0xa74a80, 0xc445b966a0, 0xc445726680, 0xc4457266a0, 0x7fad6035c8e0, 0xc449609770, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449609770, 0xa74a80, 0xc445b966a0, 0x7fad6035c8e0, 0xc449609770, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc449609770, 0xa74a80, 0xc445b966a0, 0xc445726680, 0xc4457266c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xffffffff, 0x0, 0x19, 0xc435e0b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xc445726680, 0xc4457266e0, 0xc435e0b878, 0x558012, 0x588a59f3, 0x37e5eaf5)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xed01c50f3, 0x0, 0x0, 0xdcd52)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xc445726680, 0xc445726700, 0xa91b40, 0xc445726720, 0xc445726680, 0xc445726720)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xc420246100, 0xc428dc6980, 0xc435e0ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442b52ff0, 0xa74a80, 0xc445b966a0, 0xc445726680, 0xc4457266a0, 0x5845, 0xc435e50b18, 0x0, 0xc435e50b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442b52ff0, 0xc4445a82a0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f309dfd)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064b00, 0xc420331380, 0x10265c0, 0xc442b53020, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064b00, 0xc442b53020)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438064b00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 329 [IO wait]:
net.runtime_pollWait(0x7fad6039d2a0, 0x72, 0x8)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc421d461b0, 0x72, 0xc4200458d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc421d461b0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc421d46150, 0xc421fa0000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc42019de50, 0xc421fa0000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc4210ec6c0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc4210ec6c0, 0xc420246578, 0x9, 0x9, 0x43fe80, 0xc43eaf22c0, 0xc4445b4e20)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc4210ec6c0, 0xc420246578, 0x9, 0x9, 0x9, 0xc420045ae8, 0x451a90, 0xc438650840)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc4210ec6c0, 0xc420246578, 0x9, 0x9, 0xc43eaf22c0, 0x1, 0x2)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc420246578, 0x9, 0x9, 0x102f380, 0xc4210ec6c0, 0x0, 0xc400000000, 0xc420045c80, 0x7f3105)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420246540, 0x3895, 0x3895, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc421f520f0, 0xc4445f9a70, 0xc4445f9a70, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420246600, 0xc421f52180, 0xb475b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc420246600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc42019de50, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc42019de50)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 8922 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43ba15e90, 0xc43ba15ec0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43ba15e90, 0xc43ab20000, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43ba15e90, 0xc44a3ff520, 0x0, 0xc4424c5398, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43ba15e90, 0xa74a80, 0xc44a3ff520, 0x7fad6035c8e0, 0xc43ba15e90, 0x1038500, 0xc43ba15e60)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43ba15e60, 0xa74a80, 0xc44a3ff520, 0xc43ab49be0, 0xc43ab49c00, 0x7fad6035c8e0, 0xc43ba15e60, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43ba15e60, 0xa74a80, 0xc44a3ff520, 0x7fad6035c8e0, 0xc43ba15e60, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43ba15e60, 0xa74a80, 0xc44a3ff520, 0xc43ab49be0, 0xc43ab49c20, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xffffffff, 0x0, 0x19, 0xc43ed817e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xc43ab49be0, 0xc43ab49c40, 0xc43ed81878, 0x558012, 0x588a57f5, 0x3a6fc265)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xed01c4ef5, 0x0, 0x0, 0x62db0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xc43ab49be0, 0xc43ab49c60, 0xa91b40, 0xc43ab49c80, 0xc43ab49be0, 0xc43ab49c80)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xc420246100, 0xc442f6c840, 0xc43ed81a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4445c4f00, 0xa74a80, 0xc44a3ff520, 0xc43ab49be0, 0xc43ab49c00, 0x49bc, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4445c4f00, 0xc42b6f2540, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5986c5)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b800, 0xc420331380, 0x10265c0, 0xc4445c4f30, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b800, 0xc4445c4f30)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b800)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 1182 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204962c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204962c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42cdd1d20, 0x3, 0x3, 0xc42a6df448, 0x0, 0xc42ccadac0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42cc96e70, 0x159dc604330, 0x159dc64d710, 0xc42cdd1d20, 0x3, 0x3, 0xc42cc96e00, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42cc96e70, 0xc42cc96d50, 0x0, 0xc42cdf6038, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42cc96e70, 0xa934c0, 0xc42cc96d50, 0x7fad6035c8e0, 0xc42cc96e70, 0x1038500, 0xc42cc96e40)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42cc96e40, 0xa934c0, 0xc42cc96d50, 0xc42cdd1c20, 0xc42cdd1c40, 0x7fad6035c8e0, 0xc42cc96e40, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cc96e40, 0xa934c0, 0xc42cc96d50, 0x7fad6035c8e0, 0xc42cc96e40, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42cc96e40, 0xa934c0, 0xc42cc96d50, 0xc42cdd1c20, 0xc42cdd1c60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xffffffff, 0x0, 0x19, 0xc42a6df7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xc42cdd1c20, 0xc42cdd1c80, 0xc42a6df878, 0x558012, 0x588a5743, 0x12b73ef2)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xed01c4e43, 0x0, 0x0, 0x9e8d)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xc42cdd1c20, 0xc42cdd1ca0, 0xa91b40, 0xc42cdd1cc0, 0xc42cdd1c20, 0xc42cdd1cc0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xc420246100, 0xc42cc87fe0, 0xc42a6dfa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42cc96c30, 0xa934c0, 0xc42cc96d50, 0xc42cdd1c20, 0xc42cdd1c40, 0x65, 0xc425813b18, 0x0, 0xc425813b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42cc96c30, 0xc42cb7a4e0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41fa28d46)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429a83900, 0xc420331380, 0x10265d8, 0xc42cc96cf0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429a83900, 0xc42cc96cf0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429a83900)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 871 [select, 13 minutes]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc426048300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 11636 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc447654990, 0xc4476549c0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc447654990, 0xc44289f180, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc447654990, 0xc44289e6c0, 0x0, 0xc42f98c890, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc447654990, 0xa74a80, 0xc44289e6c0, 0x7fad6035c8e0, 0xc447654990, 0x1038500, 0xc447654960)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc447654960, 0xa74a80, 0xc44289e6c0, 0xc44289f080, 0xc44289f0a0, 0x7fad6035c8e0, 0xc447654960, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447654960, 0xa74a80, 0xc44289e6c0, 0x7fad6035c8e0, 0xc447654960, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc447654960, 0xa74a80, 0xc44289e6c0, 0xc44289f080, 0xc44289f0c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xffffffff, 0x0, 0x19, 0xc439fb77e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xc44289f080, 0xc44289f0e0, 0xc439fb7878, 0x558012, 0x588a5999, 0x3aa461f1)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xed01c5099, 0x0, 0x0, 0x14ecf)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xc44289f080, 0xc44289f100, 0xa91b40, 0xc44289f120, 0xc44289f080, 0xc44289f120)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xc420246100, 0xc42f9bfc80, 0xc439fb7a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc447654420, 0xa74a80, 0xc44289e6c0, 0xc44289f080, 0xc44289f0a0, 0x859, 0xc440d62b18, 0x0, 0xc440d62b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc447654420, 0xc443093800, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f20a56f)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2800, 0xc420331380, 0x10265c0, 0xc4476544e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2800, 0xc4476544e0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2800)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8921 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4445c4030, 0xc4445c4060, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4445c4030, 0xc44a445f20, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4445c4030, 0xc44a4d05c0, 0x0, 0xc442f6c770, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4445c4030, 0xa74a80, 0xc44a4d05c0, 0x7fad6035c8e0, 0xc4445c4030, 0x1038500, 0xc4445c4000)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4445c4000, 0xa74a80, 0xc44a4d05c0, 0xc44a445e20, 0xc44a445e40, 0x7fad6035c8e0, 0xc4445c4000, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4445c4000, 0xa74a80, 0xc44a4d05c0, 0x7fad6035c8e0, 0xc4445c4000, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4445c4000, 0xa74a80, 0xc44a4d05c0, 0xc44a445e20, 0xc44a445e60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xffffffff, 0x0, 0x19, 0xc43ed7f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xc44a445e20, 0xc44a445e80, 0xc43ed7f878, 0x558012, 0x588a57f5, 0x39fd530a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xed01c4ef5, 0x0, 0x0, 0x47d77)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xc44a445e20, 0xc44a445ea0, 0xa91b40, 0xc44a445ec0, 0xc44a445e20, 0xc44a445ec0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xc420246100, 0xc44479a890, 0xc43ed7fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43a29d440, 0xa74a80, 0xc44a4d05c0, 0xc44a445e20, 0xc44a445e40, 0x31bd, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43a29d440, 0xc42b6f23c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5986c5)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b500, 0xc420331380, 0x10265c0, 0xc43a29d470, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b500, 0xc43a29d470)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5b500)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10275 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43562b590, 0xc43562b5f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43562b590, 0xc43ee00c60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43562b590, 0xc43b3f6e40, 0x0, 0xc436f58488, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43562b590, 0xa74a80, 0xc43b3f6e40, 0x7fad6035c8e0, 0xc43562b590, 0x1038500, 0xc43562b560)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43562b560, 0xa74a80, 0xc43b3f6e40, 0xc43ee00b40, 0xc43ee00b60, 0x7fad6035c8e0, 0xc43562b560, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43562b560, 0xa74a80, 0xc43b3f6e40, 0x7fad6035c8e0, 0xc43562b560, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43562b560, 0xa74a80, 0xc43b3f6e40, 0xc43ee00b40, 0xc43ee00b80, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xffffffff, 0x0, 0x19, 0xc442c197e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xc43ee00b40, 0xc43ee00ba0, 0xc442c19878, 0x558012, 0x588a58c7, 0x3b8f70bd)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xed01c4fc7, 0x0, 0x0, 0x12a6b9)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xc43ee00b40, 0xc43ee00bc0, 0xa91b40, 0xc43ee00be0, 0xc43ee00b40, 0xc43ee00be0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xc420246100, 0xc4370bdf00, 0xc442c19a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43564ef00, 0xa74a80, 0xc43b3f6e40, 0xc43ee00b40, 0xc43ee00b60, 0x97be, 0xc420319b18, 0x0, 0xc420319b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43564ef00, 0xc42fa64240, 0xc420179d60, 0x0, 0x0, 0x10, 0xc43e5a21d8)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdca00, 0xc420331380, 0x10265c0, 0xc43564ef60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdca00, 0xc43564ef60)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdca00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9249 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc42a8fcf00, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f
github.com/weaveworks/cortex/ingester.(*Ingester).LabelValues(0xc4201e3040, 0x7fad6035c8e0, 0xc42a8fcf00, 0xc43f717bb0, 0x0, 0xc43f717c68, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:298 +0x60
github.com/weaveworks/cortex._Ingester_LabelValues_Handler.func1(0x7fad6035c8e0, 0xc42a8fcf00, 0xa73d60, 0xc43f717bb0, 0x7fad6035c8e0, 0xc42a8fcf00, 0x1038500, 0xc42a8fced0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:457 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a8fced0, 0xa73d60, 0xc43f717bb0, 0xc43698e580, 0xc43698e5a0, 0x7fad6035c8e0, 0xc42a8fced0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fced0, 0xa73d60, 0xc43f717bb0, 0x7fad6035c8e0, 0xc42a8fced0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a8fced0, 0xa73d60, 0xc43f717bb0, 0xc43698e580, 0xc43698e5c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0xffffffff, 0x0, 0x19, 0xc4441d97e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0xc43698e580, 0xc43698e5e0, 0xc4441d9878, 0x558012, 0x588a5827, 0xf0d8b43)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0x41167e, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0xc43698e580, 0xc43698e600, 0xa91b40, 0xc43698e640, 0xc43698e580, 0xc43698e640)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0xc420246100, 0xc43f717b70, 0xc4441d9a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a8fcdb0, 0xa73d60, 0xc43f717bb0, 0xc43698e580, 0xc43698e5a0, 0xa, 0xc42023bb18, 0x0, 0xc42023bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_LabelValues_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a8fcdb0, 0xc4415b5320, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:459 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc4390fb200, 0xc420331380, 0x10265f0, 0xc42a8fce70, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc4390fb200, 0xc42a8fce70)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc440f5ebb0, 0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc4390fb200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 12196 [chan receive, 2 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc4485843c0, 0xc43cb882d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc4485843c0, 0xc43cb882d0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc4485843c0, 0xc43cb882d0)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc441f95760, 0xc43cb882d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc441f95760, 0xc43cb882d0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc441f95760, 0xc43cb882d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc441f95740, 0xc43cb880f0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc441f95740, 0xc43cb880f0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc441f95720, 0xc43cb880f0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc441f95720, 0xc43cb880f0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc42bd16340, 0xc43cb880f0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc42bd16340, 0xc43cb88000)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc42bd16340, 0xc43cb88000)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc42854fb00, 0x1038440, 0xc438720880)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 8473 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a
github.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc429a59aa0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc42bd13430, 0xc429a59aa0, 0x10343c0, 0xc4201e3040)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326

goroutine 1271 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204895c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204895c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x67d0eb2b1a80b1fc)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42d39dc80, 0xc42939e0cd, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d39dc50, 0xc42d39dc80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d39dc50, 0xc42d4c9240, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d39dc50, 0xc42d4c9080, 0x0, 0xc42d2c7f90, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d39dc50, 0xa74a80, 0xc42d4c9080, 0x7fad6035c8e0, 0xc42d39dc50, 0x1038500, 0xc42d39dc20)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d39dc20, 0xa74a80, 0xc42d4c9080, 0xc42d4c9140, 0xc42d4c9160, 0x7fad6035c8e0, 0xc42d39dc20, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d39dc20, 0xa74a80, 0xc42d4c9080, 0x7fad6035c8e0, 0xc42d39dc20, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d39dc20, 0xa74a80, 0xc42d4c9080, 0xc42d4c9140, 0xc42d4c9180, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xffffffff, 0x0, 0x19, 0xc4215db7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xc42d4c9140, 0xc42d4c91a0, 0xc4215db878, 0x558012, 0x588a5743, 0x2059cadd)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xed01c4e43, 0x0, 0x0, 0x7bb2)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xc42d4c9140, 0xc42d4c91c0, 0xa91b40, 0xc42d4c91e0, 0xc42d4c9140, 0xc42d4c91e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xc420246100, 0xc42d2c7f20, 0xc4215dba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d39daa0, 0xa74a80, 0xc42d4c9080, 0xc42d4c9140, 0xc42d4c9160, 0x73, 0xc42023eb18, 0x0, 0xc42023eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d39daa0, 0xc42d524000, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d3b0300, 0xc420331380, 0x10265c0, 0xc42d39db60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d3b0300, 0xc42d39db60)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d3b0300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8327 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4292daa50, 0xc4292daa80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4292daa50, 0xc437b61380, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4292daa50, 0xc4383a2080, 0x0, 0xc4292d99f0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4292daa50, 0xa74a80, 0xc4383a2080, 0x7fad6035c8e0, 0xc4292daa50, 0x1038500, 0xc4292daa20)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4292daa20, 0xa74a80, 0xc4383a2080, 0xc437b61260, 0xc437b61280, 0x7fad6035c8e0, 0xc4292daa20, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4292daa20, 0xa74a80, 0xc4383a2080, 0x7fad6035c8e0, 0xc4292daa20, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4292daa20, 0xa74a80, 0xc4383a2080, 0xc437b61260, 0xc437b612a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xffffffff, 0x0, 0x19, 0xc439ee17e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xc437b61260, 0xc437b612c0, 0xc439ee1878, 0x558012, 0x588a579b, 0x32eefbdf)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xed01c4e9b, 0x0, 0x0, 0x11936e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xc437b61260, 0xc437b612e0, 0xa91b40, 0xc437b61300, 0xc437b61260, 0xc437b61300)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xc420246100, 0xc440d25d10, 0xc439ee1a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4223bde30, 0xa74a80, 0xc4383a2080, 0xc437b61260, 0xc437b61280, 0x9dd5, 0xc420e1db18, 0x0, 0xc420e1db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4223bde30, 0xc44088e0c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f40378e)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc426516400, 0xc420331380, 0x10265c0, 0xc4223bde60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc426516400, 0xc4223bde60)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc426516400)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 1019 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204962c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204962c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc423bd8e80, 0x4, 0x4, 0xc4293e3448, 0x0, 0xc42a49fac0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42a49df80, 0x159dc604330, 0x159dc64d710, 0xc423bd8e80, 0x4, 0x4, 0xc42a49df00, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42a49df80, 0xc42a49de30, 0x0, 0xc42a4d2700, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42a49df80, 0xa934c0, 0xc42a49de30, 0x7fad6035c8e0, 0xc42a49df80, 0x1038500, 0xc42a49df50)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a49df50, 0xa934c0, 0xc42a49de30, 0xc423bd8c00, 0xc423bd8c60, 0x7fad6035c8e0, 0xc42a49df50, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a49df50, 0xa934c0, 0xc42a49de30, 0x7fad6035c8e0, 0xc42a49df50, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a49df50, 0xa934c0, 0xc42a49de30, 0xc423bd8c00, 0xc423bd8ca0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xffffffff, 0x0, 0x19, 0xc4293e37e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xc423bd8c00, 0xc423bd8ce0, 0xc4293e3878, 0x558012, 0x588a5742, 0x166f6bd)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xed01c4e42, 0x0, 0x0, 0xdea7)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xc423bd8c00, 0xc423bd8d20, 0xa91b40, 0xc423bd8d60, 0xc423bd8c00, 0xc423bd8d60)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xc420246100, 0xc42a4d2690, 0xc4293e3a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a49dd10, 0xa934c0, 0xc42a49de30, 0xc423bd8c00, 0xc423bd8c60, 0x83, 0xc420317b18, 0x0, 0xc420317b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a49dd10, 0xc429975f20, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc4204ffe00, 0xc420331380, 0x10265d8, 0xc42a49ddd0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc4204ffe00, 0xc42a49ddd0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc4204ffe00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 941 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048e904)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048e900)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x9c30b4199368dad5)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a0776b0, 0xc420162418, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a076540, 0xc42a0776b0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a076540, 0xc425157e20, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a076540, 0xc425d81580, 0x0, 0xc42a074e78, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a076540, 0xa74a80, 0xc425d81580, 0x7fad6035c8e0, 0xc42a076540, 0x1038500, 0xc42a076510)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a076510, 0xa74a80, 0xc425d81580, 0xc4251562a0, 0xc4251562e0, 0x7fad6035c8e0, 0xc42a076510, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a076510, 0xa74a80, 0xc425d81580, 0x7fad6035c8e0, 0xc42a076510, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a076510, 0xa74a80, 0xc425d81580, 0xc4251562a0, 0xc425156320, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xffffffff, 0x0, 0x19, 0xc4215b97e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xc4251562a0, 0xc425156360, 0xc4215b9878, 0x558012, 0x588a5741, 0x35add981)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xed01c4e41, 0x0, 0x0, 0x27b95e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xc4251562a0, 0xc4251563a0, 0xa91b40, 0xc4251563e0, 0xc4251562a0, 0xc4251563e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xc420246100, 0xc4266315b0, 0xc4215b9a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc429fabd10, 0xa74a80, 0xc425d81580, 0xc4251562a0, 0xc4251562e0, 0x85f8, 0xc42023bb18, 0x0, 0xc42023bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc429fabd10, 0xc4294e3260, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0800, 0xc420331380, 0x10265c0, 0xc429fabd40, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0800, 0xc429fabd40)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0800)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8737 [chan send, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc44776e870, 0xc443c34180, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 12195 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43cf98450, 0xc43cf98480, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43cf98450, 0xc4400f5f40, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43cf98450, 0xc4457276a0, 0x0, 0xc4231ee1f8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43cf98450, 0xa74a80, 0xc4457276a0, 0x7fad6035c8e0, 0xc43cf98450, 0x1038500, 0xc43cf98420)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43cf98420, 0xa74a80, 0xc4457276a0, 0xc4400f5e20, 0xc4400f5e40, 0x7fad6035c8e0, 0xc43cf98420, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43cf98420, 0xa74a80, 0xc4457276a0, 0x7fad6035c8e0, 0xc43cf98420, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43cf98420, 0xa74a80, 0xc4457276a0, 0xc4400f5e20, 0xc4400f5e60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xffffffff, 0x0, 0x19, 0xc4359277e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xc4400f5e20, 0xc4400f5ea0, 0xc435927878, 0x558012, 0x588a59f4, 0x99738f8)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xed01c50f4, 0x0, 0x0, 0x140cd0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xc4400f5e20, 0xc4400f5ec0, 0xa91b40, 0xc4400f5ee0, 0xc4400f5e20, 0xc4400f5ee0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xc420246100, 0xc427f8d760, 0xc435927a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc449610d80, 0xa74a80, 0xc4457276a0, 0xc4400f5e20, 0xc4400f5e40, 0x7532, 0xc435e51b18, 0x0, 0xc435e51b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc449610d80, 0xc4445a8480, 0xc420179d60, 0x0, 0x0, 0x2b9de3948fa81040, 0x720a0a5b0a02cd0a)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065500, 0xc420331380, 0x10265c0, 0xc449610db0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065500, 0xc449610db0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065500)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 993 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048e0c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048e0c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x8ca9f4c8f66dcb26)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a1a19e0, 0xc420162418, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a1a17d0, 0xc42a1a19e0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a1a17d0, 0xc4244c0940, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a1a17d0, 0xc425120e20, 0x0, 0xc42a1b7488, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a1a17d0, 0xa74a80, 0xc425120e20, 0x7fad6035c8e0, 0xc42a1a17d0, 0x1038500, 0xc42a1a17a0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a1a17a0, 0xa74a80, 0xc425120e20, 0xc4244c05e0, 0xc4244c0600, 0x7fad6035c8e0, 0xc42a1a17a0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a1a17a0, 0xa74a80, 0xc425120e20, 0x7fad6035c8e0, 0xc42a1a17a0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a1a17a0, 0xa74a80, 0xc425120e20, 0xc4244c05e0, 0xc4244c0620, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xffffffff, 0x0, 0x19, 0xc42910f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xc4244c05e0, 0xc4244c0640, 0xc42910f878, 0x558012, 0x588a5741, 0x36482006)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xed01c4e41, 0x0, 0x0, 0x825ce)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xc4244c05e0, 0xc4244c0680, 0xa91b40, 0xc4244c06a0, 0xc4244c05e0, 0xc4244c06a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xc420246100, 0xc429fad030, 0xc42910fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a196120, 0xa74a80, 0xc425120e20, 0xc4244c05e0, 0xc4244c0600, 0x6efd, 0xc427ba0b18, 0x0, 0xc427ba0b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a196120, 0xc429d97080, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4207b3300, 0xc420331380, 0x10265c0, 0xc42a196150, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4207b3300, 0xc42a196150)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4207b3300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 1190 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204962c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204962c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42c9402f0, 0x2, 0x2, 0xc42a6e3448, 0x0, 0xc42c9371c0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42c9452f0, 0x159dc604330, 0x159dc64d710, 0xc42c9402f0, 0x2, 0x2, 0xc42c945200, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42c9452f0, 0xc42c945200, 0x0, 0xc42c9402c8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42c9452f0, 0xa934c0, 0xc42c945200, 0x7fad6035c8e0, 0xc42c9452f0, 0x1038500, 0xc42c9452c0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c9452c0, 0xa934c0, 0xc42c945200, 0xc42c946f80, 0xc42c946fa0, 0x7fad6035c8e0, 0xc42c9452c0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9452c0, 0xa934c0, 0xc42c945200, 0x7fad6035c8e0, 0xc42c9452c0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c9452c0, 0xa934c0, 0xc42c945200, 0xc42c946f80, 0xc42c946fc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xffffffff, 0x0, 0x19, 0xc42a6e37e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xc42c946f80, 0xc42c946fe0, 0xc42a6e3878, 0x558012, 0x588a5743, 0x8c40078)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xc42a6e3900, 0x0, 0x0, 0x7f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xc42c946f80, 0xc42c947000, 0xa91b40, 0xc42c947020, 0xc42c946f80, 0xc42c947020)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xc420246100, 0xc42c940280, 0xc42a6e3a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42c9450e0, 0xa934c0, 0xc42c945200, 0xc42c946f80, 0xc42c946fa0, 0x4e, 0xc420238b18, 0x0, 0xc420238b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42c9450e0, 0xc42ca5c060, 0xc420179d60, 0x0, 0x0, 0xc42ad6bfa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc428c67b00, 0xc420331380, 0x10265d8, 0xc42c9451a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc428c67b00, 0xc42c9451a0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc428c67b00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8990 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc442348510, 0xc442348540, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc442348510, 0xc435ba7860, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc442348510, 0xc44693fdc0, 0x0, 0xc43c1e6de8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc442348510, 0xa74a80, 0xc44693fdc0, 0x7fad6035c8e0, 0xc442348510, 0x1038500, 0xc4423484e0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4423484e0, 0xa74a80, 0xc44693fdc0, 0xc435ba7740, 0xc435ba7760, 0x7fad6035c8e0, 0xc4423484e0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4423484e0, 0xa74a80, 0xc44693fdc0, 0x7fad6035c8e0, 0xc4423484e0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4423484e0, 0xa74a80, 0xc44693fdc0, 0xc435ba7740, 0xc435ba77a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xffffffff, 0x0, 0x19, 0xc44a4917e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xc435ba7740, 0xc435ba77c0, 0xc44a491878, 0x558012, 0x588a5814, 0x868da4c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xc44a491900, 0x0, 0x0, 0xbf)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xc435ba7740, 0xc435ba77e0, 0xa91b40, 0xc435ba7800, 0xc435ba7740, 0xc435ba7800)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xc420246100, 0xc448040e30, 0xc44a491a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc441afb830, 0xa74a80, 0xc44693fdc0, 0xc435ba7740, 0xc435ba7760, 0x33e1, 0xc420e1db18, 0x0, 0xc420e1db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc441afb830, 0xc443c35f20, 0xc420179d60, 0x0, 0x0, 0xc420252800, 0x450280)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1d00, 0xc420331380, 0x10265c0, 0xc441afb8f0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1d00, 0xc441afb8f0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1d00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8916 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43c59b5c0, 0xc43c59b5f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43c59b5c0, 0xc44197c5a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43c59b5c0, 0xc43ecd1c00, 0x0, 0xc42df97b90, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43c59b5c0, 0xa74a80, 0xc43ecd1c00, 0x7fad6035c8e0, 0xc43c59b5c0, 0x1038500, 0xc43c59b590)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43c59b590, 0xa74a80, 0xc43ecd1c00, 0xc44197c4a0, 0xc44197c4c0, 0x7fad6035c8e0, 0xc43c59b590, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c59b590, 0xa74a80, 0xc43ecd1c00, 0x7fad6035c8e0, 0xc43c59b590, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43c59b590, 0xa74a80, 0xc43ecd1c00, 0xc44197c4a0, 0xc44197c4e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xffffffff, 0x0, 0x19, 0xc44abab7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xc44197c4a0, 0xc44197c500, 0xc44abab878, 0x558012, 0x588a57f2, 0xed7e733)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xed01c4ef2, 0x0, 0x0, 0x51cec9)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xc44197c4a0, 0xc44197c520, 0xa91b40, 0xc44197c540, 0xc44197c4a0, 0xc44197c540)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xc420246100, 0xc44609cc80, 0xc44ababa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442d73b00, 0xa74a80, 0xc43ecd1c00, 0xc44197c4a0, 0xc44197c4c0, 0x81a7, 0xc420049b18, 0x0, 0xc420049b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442d73b00, 0xc442ba8ea0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f0978d1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5a700, 0xc420331380, 0x10265c0, 0xc442d73b30, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5a700, 0xc442d73b30)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439a5a700)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8681 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a
github.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc436c44600)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc4386531d0, 0xc436c44600, 0x10343c0, 0xc4201e3040)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326

goroutine 8225 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42ce529f0, 0xc42ce52a20, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42ce529f0, 0xc4379f4b40, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42ce529f0, 0xc437a66c60, 0x0, 0xc42ce48808, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42ce529f0, 0xa74a80, 0xc437a66c60, 0x7fad6035c8e0, 0xc42ce529f0, 0x1038500, 0xc42ce529c0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42ce529c0, 0xa74a80, 0xc437a66c60, 0xc4379f4a00, 0xc4379f4a20, 0x7fad6035c8e0, 0xc42ce529c0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42ce529c0, 0xa74a80, 0xc437a66c60, 0x7fad6035c8e0, 0xc42ce529c0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42ce529c0, 0xa74a80, 0xc437a66c60, 0xc4379f4a00, 0xc4379f4a40, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0xffffffff, 0x0, 0x19, 0xc43a66b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0xc4379f4a00, 0xc4379f4a80, 0xc43a66b878, 0x558012, 0x588a579b, 0x36f30836)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0x411715, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0xc4379f4a00, 0xc4379f4aa0, 0xa91b40, 0xc4379f4ac0, 0xc4379f4a00, 0xc4379f4ac0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0xc420246100, 0xc42d311090, 0xc43a66ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc429c871d0, 0xa74a80, 0xc437a66c60, 0xc4379f4a00, 0xc4379f4a20, 0x8844, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc429c871d0, 0xc42ba4d260, 0xc420179d60, 0x0, 0x0, 0xc43ceb47a8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42cbace00, 0xc420331380, 0x10265c0, 0xc429c87200, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42cbace00, 0xc429c87200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42cbace00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 961 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc420497484)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc420497480)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x30c7ec3b1c6a8f83)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a4fcd80, 0xc420162418, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a4fc600, 0xc42a4fcd80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a4fc600, 0xc423c7b2c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a4fc600, 0xc423d58e80, 0x0, 0xc42a4fb680, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a4fc600, 0xa74a80, 0xc423d58e80, 0x7fad6035c8e0, 0xc42a4fc600, 0x1038500, 0xc42a4fc5d0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a4fc5d0, 0xa74a80, 0xc423d58e80, 0xc423c7a000, 0xc423c7a040, 0x7fad6035c8e0, 0xc42a4fc5d0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4fc5d0, 0xa74a80, 0xc423d58e80, 0x7fad6035c8e0, 0xc42a4fc5d0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a4fc5d0, 0xa74a80, 0xc423d58e80, 0xc423c7a000, 0xc423c7a080, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xffffffff, 0x0, 0x19, 0xc4260f57e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xc423c7a000, 0xc423c7a0e0, 0xc4260f5878, 0x558012, 0x588a5741, 0x3a26f8f6)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xed01c4e41, 0x0, 0x0, 0xf6ff9)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xc423c7a000, 0xc423c7a180, 0xa91b40, 0xc423c7a1c0, 0xc423c7a000, 0xc423c7a1c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xc420246100, 0xc42a075ba0, 0xc4260f5a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a4d8150, 0xa74a80, 0xc423d58e80, 0xc423c7a000, 0xc423c7a040, 0xa7b1, 0xc420319b18, 0x0, 0xc420319b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a4d8150, 0xc42a1de300, 0xc420179d60, 0x0, 0x0, 0x312d70691c126564, 0x2d312d30322d3237)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1600, 0xc420331380, 0x10265c0, 0xc42a4d8180, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1600, 0xc42a4d8180)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 929 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048df04)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048df00)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0xe11e7b125b1afd74)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d1342f0, 0x2, 0x2, 0xc42d1f7448, 0x0, 0xc42d1363c0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1e2750, 0x159dc604330, 0x159dc64d710, 0xc42d1342f0, 0x2, 0x2, 0xc42d1e2700, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1e2750, 0xc42d1e2690, 0x0, 0xc42d1342c8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d1e2750, 0xa934c0, 0xc42d1e2690, 0x7fad6035c8e0, 0xc42d1e2750, 0x1038500, 0xc42d1e2720)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d1e2720, 0xa934c0, 0xc42d1e2690, 0xc42d12a600, 0xc42d12a620, 0x7fad6035c8e0, 0xc42d1e2720, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1e2720, 0xa934c0, 0xc42d1e2690, 0x7fad6035c8e0, 0xc42d1e2720, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d1e2720, 0xa934c0, 0xc42d1e2690, 0xc42d12a600, 0xc42d12a640, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xffffffff, 0x0, 0x19, 0xc42d1f77e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xc42d12a600, 0xc42d12a660, 0xc42d1f7878, 0x558012, 0x588a5743, 0x1a23da29)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xed01c4e43, 0x0, 0x0, 0x528a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xc42d12a600, 0xc42d12a680, 0xa91b40, 0xc42d12a6a0, 0xc42d12a600, 0xc42d12a6a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xc420246100, 0xc42d134280, 0xc42d1f7a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d1e2570, 0xa934c0, 0xc42d1e2690, 0xc42d12a600, 0xc42d12a620, 0x43, 0xc420314b18, 0x0, 0xc420314b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d1e2570, 0xc42d1ea360, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ec100, 0xc420331380, 0x10265d8, 0xc42d1e2630, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ec100, 0xc42d1e2630)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ec100)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 958 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048e0c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048e0c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x5080f3cc323e99c)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a2011d0, 0xc422334c2d, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a200570, 0xc42a2011d0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a200570, 0xc4243f37c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a200570, 0xc4244b0920, 0x0, 0xc42a204508, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a200570, 0xa74a80, 0xc4244b0920, 0x7fad6035c8e0, 0xc42a200570, 0x1038500, 0xc42a200540)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a200540, 0xa74a80, 0xc4244b0920, 0xc4243f2a20, 0xc4243f2a40, 0x7fad6035c8e0, 0xc42a200540, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a200540, 0xa74a80, 0xc4244b0920, 0x7fad6035c8e0, 0xc42a200540, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a200540, 0xa74a80, 0xc4244b0920, 0xc4243f2a20, 0xc4243f2a60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xffffffff, 0x0, 0x19, 0xc4200577e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xc4243f2a20, 0xc4243f2a80, 0xc420057878, 0x558012, 0x588a5741, 0x36f3bd26)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xed01c4e41, 0x0, 0x0, 0x1d184f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xc4243f2a20, 0xc4243f2ac0, 0xa91b40, 0xc4243f2ae0, 0xc4243f2a20, 0xc4243f2ae0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xc420246100, 0xc42a15cc20, 0xc420057a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a0779e0, 0xa74a80, 0xc4244b0920, 0xc4243f2a20, 0xc4243f2a40, 0xaa72, 0xc42023bb18, 0x0, 0xc42023bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a0779e0, 0xc4288218c0, 0xc420179d60, 0x0, 0x0, 0x312d70691c126564, 0x2d312d30322d3237)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421ed1300, 0xc420331380, 0x10265c0, 0xc42a077a10, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421ed1300, 0xc42a077a10)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421ed1300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 999 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048df04)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048df00)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x3bd0391af88994ef)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc429e0f8c0, 0xc422334c2d, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc429e0ec60, 0xc429e0f8c0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc429e0ec60, 0xc42559ce80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc429e0ec60, 0xc4256362a0, 0x0, 0xc429e11170, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc429e0ec60, 0xa74a80, 0xc4256362a0, 0x7fad6035c8e0, 0xc429e0ec60, 0x1038500, 0xc429e0ec30)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc429e0ec30, 0xa74a80, 0xc4256362a0, 0xc42559c2c0, 0xc42559c340, 0x7fad6035c8e0, 0xc429e0ec30, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429e0ec30, 0xa74a80, 0xc4256362a0, 0x7fad6035c8e0, 0xc429e0ec30, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc429e0ec30, 0xa74a80, 0xc4256362a0, 0xc42559c2c0, 0xc42559c380, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xffffffff, 0x0, 0x19, 0xc4215b77e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xc42559c2c0, 0xc42559c3a0, 0xc4215b7878, 0x558012, 0x588a5741, 0x333ed171)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xed01c4e41, 0x0, 0x0, 0x12f6df)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xc42559c2c0, 0xc42559c3c0, 0xa91b40, 0xc42559c3e0, 0xc42559c2c0, 0xc42559c3e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xc420246100, 0xc429d4f450, 0xc4215b7a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc429dd1bc0, 0xa74a80, 0xc4256362a0, 0xc42559c2c0, 0xc42559c340, 0xb217, 0xc42023db18, 0x0, 0xc42023db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc429dd1bc0, 0xc429d96060, 0xc420179d60, 0x0, 0x0, 0x1265646f6e040a24, 0x2d3237312d70691c)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb5600, 0xc420331380, 0x10265c0, 0xc429dd1bf0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb5600, 0xc429dd1bf0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb5600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 12390 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43bef8a80, 0xc43bef8ab0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43bef8a80, 0xc445db5160, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43bef8a80, 0xc43a890080, 0x0, 0xc43ce48d30, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43bef8a80, 0xa74a80, 0xc43a890080, 0x7fad6035c8e0, 0xc43bef8a80, 0x1038500, 0xc43bef8a50)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43bef8a50, 0xa74a80, 0xc43a890080, 0xc445db5060, 0xc445db5080, 0x7fad6035c8e0, 0xc43bef8a50, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43bef8a50, 0xa74a80, 0xc43a890080, 0x7fad6035c8e0, 0xc43bef8a50, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43bef8a50, 0xa74a80, 0xc43a890080, 0xc445db5060, 0xc445db50a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xffffffff, 0x0, 0x19, 0xc4377e37e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xc445db5060, 0xc445db50c0, 0xc4377e3878, 0x558012, 0x588a5a0e, 0x11403c0f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xed01c510e, 0x0, 0x0, 0x1ed2d7)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xc445db5060, 0xc445db50e0, 0xa91b40, 0xc445db5100, 0xc445db5060, 0xc445db5100)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xc420246100, 0xc44930f9c0, 0xc4377e3a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43dce3920, 0xa74a80, 0xc43a890080, 0xc445db5060, 0xc445db5080, 0x7b07, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43dce3920, 0xc4377bdb00, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1600, 0xc420331380, 0x10265c0, 0xc43dce3950, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1600, 0xc43dce3950)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 11634 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441572de0, 0xc441572e10, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441572de0, 0xc4445abb60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441572de0, 0xc446c1b160, 0x0, 0xc42fb10068, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441572de0, 0xa74a80, 0xc446c1b160, 0x7fad6035c8e0, 0xc441572de0, 0x1038500, 0xc441572db0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc441572db0, 0xa74a80, 0xc446c1b160, 0xc4445aba40, 0xc4445aba60, 0x7fad6035c8e0, 0xc441572db0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441572db0, 0xa74a80, 0xc446c1b160, 0x7fad6035c8e0, 0xc441572db0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc441572db0, 0xa74a80, 0xc446c1b160, 0xc4445aba40, 0xc4445aba80, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xffffffff, 0x0, 0x19, 0xc439fbb7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xc4445aba40, 0xc4445abaa0, 0xc439fbb878, 0x558012, 0x588a5999, 0x3994a188)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xed01c5099, 0x0, 0x0, 0x4c5e6)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xc4445aba40, 0xc4445abac0, 0xa91b40, 0xc4445abb00, 0xc4445aba40, 0xc4445abb00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xc420246100, 0xc42fb46c20, 0xc439fbba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4415721e0, 0xa74a80, 0xc446c1b160, 0xc4445aba40, 0xc4445aba60, 0x2c05, 0xc440d62b18, 0x0, 0xc440d62b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4415721e0, 0xc4430930e0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f20a56f)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2200, 0xc420331380, 0x10265c0, 0xc441572210, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2200, 0xc441572210)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4341d2200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9106 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43b08ef60, 0xc43b08ef90, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43b08ef60, 0xc441178c20, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43b08ef60, 0xc447e1b140, 0x0, 0xc440f5f090, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43b08ef60, 0xa74a80, 0xc447e1b140, 0x7fad6035c8e0, 0xc43b08ef60, 0x1038500, 0xc43b08ef30)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43b08ef30, 0xa74a80, 0xc447e1b140, 0xc441178b20, 0xc441178b40, 0x7fad6035c8e0, 0xc43b08ef30, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b08ef30, 0xa74a80, 0xc447e1b140, 0x7fad6035c8e0, 0xc43b08ef30, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43b08ef30, 0xa74a80, 0xc447e1b140, 0xc441178b20, 0xc441178b60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xffffffff, 0x0, 0x19, 0xc446a2d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xc441178b20, 0xc441178b80, 0xc446a2d878, 0x558012, 0x588a5813, 0x38229eae)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xed01c4f13, 0x0, 0x0, 0x5a1fa)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xc441178b20, 0xc441178ba0, 0xa91b40, 0xc441178bc0, 0xc441178b20, 0xc441178bc0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xc420246100, 0xc441a50070, 0xc446a2da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43d26de90, 0xa74a80, 0xc447e1b140, 0xc441178b20, 0xc441178b40, 0x290d, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43d26de90, 0xc4431f7e60, 0xc420179d60, 0x0, 0x0, 0xc420236800, 0x450280)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b4e00, 0xc420331380, 0x10265c0, 0xc43d26dec0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b4e00, 0xc43d26dec0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b4e00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 960 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048eb04)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048eb00)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x8b63b2637083582)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a336ff0, 0xc420162418, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a336c00, 0xc42a336ff0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a336c00, 0xc4240defa0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a336c00, 0xc4241f0d60, 0x0, 0xc42a33c970, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a336c00, 0xa74a80, 0xc4241f0d60, 0x7fad6035c8e0, 0xc42a336c00, 0x1038500, 0xc42a336bd0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a336bd0, 0xa74a80, 0xc4241f0d60, 0xc4240dea80, 0xc4240deaa0, 0x7fad6035c8e0, 0xc42a336bd0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a336bd0, 0xa74a80, 0xc4241f0d60, 0x7fad6035c8e0, 0xc42a336bd0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a336bd0, 0xa74a80, 0xc4241f0d60, 0xc4240dea80, 0xc4240deac0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xffffffff, 0x0, 0x19, 0xc4215dd7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xc4240dea80, 0xc4240deae0, 0xc4215dd878, 0x558012, 0x588a5741, 0x378ba2ff)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xed01c4e41, 0x0, 0x0, 0x1633d4)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xc4240dea80, 0xc4240deb00, 0xa91b40, 0xc4240deb40, 0xc4240dea80, 0xc4240deb40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xc420246100, 0xc42a2a6aa0, 0xc4215dda70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a077f20, 0xa74a80, 0xc4241f0d60, 0xc4240dea80, 0xc4240deaa0, 0xbe4a, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a077f20, 0xc42a1fed80, 0xc420179d60, 0x0, 0x0, 0x312d70691c126564, 0x2d312d30322d3237)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1500, 0xc420331380, 0x10265c0, 0xc42a077f50, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1500, 0xc42a077f50)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1500)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 1013 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048e0c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048e0c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x59362661c4be275c)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc42a1598f0, 0xc422334c2d, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a159560, 0xc42a1598f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a159560, 0xc424576380, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a159560, 0xc425337d40, 0x0, 0xc42a15c850, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a159560, 0xa74a80, 0xc425337d40, 0x7fad6035c8e0, 0xc42a159560, 0x1038500, 0xc42a159530)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a159530, 0xa74a80, 0xc425337d40, 0xc424581d00, 0xc424581d20, 0x7fad6035c8e0, 0xc42a159530, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a159530, 0xa74a80, 0xc425337d40, 0x7fad6035c8e0, 0xc42a159530, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a159530, 0xa74a80, 0xc425337d40, 0xc424581d00, 0xc424581d60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xffffffff, 0x0, 0x19, 0xc42005b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xc424581d00, 0xc424581d80, 0xc42005b878, 0x558012, 0x588a5741, 0x36275cc8)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xed01c4e41, 0x0, 0x0, 0x634c03)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xc424581d00, 0xc424581da0, 0xa91b40, 0xc424581de0, 0xc424581d00, 0xc424581de0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xc420246100, 0xc429eba670, 0xc42005ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a066510, 0xa74a80, 0xc425337d40, 0xc424581d00, 0xc424581d20, 0x6671, 0xc427ba0b18, 0x0, 0xc427ba0b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a066510, 0xc428821260, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4204ff000, 0xc420331380, 0x10265c0, 0xc42a066540, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4204ff000, 0xc42a066540)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4204ff000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8818 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc44930ea6c)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc44930ea60)
	/usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc44930ea60, 0xc43cdd9380)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e

goroutine 8420 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43ffd7b60, 0xc43ffd7b90, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43ffd7b60, 0xc432956340, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43ffd7b60, 0xc43297ad60, 0x0, 0xc43c974718, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43ffd7b60, 0xa74a80, 0xc43297ad60, 0x7fad6035c8e0, 0xc43ffd7b60, 0x1038500, 0xc43ffd7b30)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43ffd7b30, 0xa74a80, 0xc43297ad60, 0xc4329561e0, 0xc432956220, 0x7fad6035c8e0, 0xc43ffd7b30, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43ffd7b30, 0xa74a80, 0xc43297ad60, 0x7fad6035c8e0, 0xc43ffd7b30, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43ffd7b30, 0xa74a80, 0xc43297ad60, 0xc4329561e0, 0xc432956260, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xffffffff, 0x0, 0x19, 0xc42ee537e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xc4329561e0, 0xc432956280, 0xc42ee53878, 0x558012, 0x588a57d4, 0xe57917e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xed01c4ed4, 0x0, 0x0, 0xbf481)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xc4329561e0, 0xc4329562a0, 0xa91b40, 0xc4329562c0, 0xc4329561e0, 0xc4329562c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xc420246100, 0xc4495f2940, 0xc42ee53a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43b859230, 0xa74a80, 0xc43297ad60, 0xc4329561e0, 0xc432956220, 0x530e, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43b859230, 0xc44661b080, 0xc420179d60, 0x0, 0x0, 0xc43b43a600, 0xb)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421002000, 0xc420331380, 0x10265c0, 0xc43b859260, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421002000, 0xc43b859260)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc421002000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 1286 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204962c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204962c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d96b7e0, 0x2, 0x2, 0xc42d711448, 0x0, 0xc42db9fa00)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d96d3b0, 0x159dc604330, 0x159dc64d710, 0xc42d96b7e0, 0x2, 0x2, 0xc42d96d300, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d96d3b0, 0xc42d96d2c0, 0x0, 0xc42d96b7b8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d96d3b0, 0xa934c0, 0xc42d96d2c0, 0x7fad6035c8e0, 0xc42d96d3b0, 0x1038500, 0xc42d96d380)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d96d380, 0xa934c0, 0xc42d96d2c0, 0xc42dbb7fc0, 0xc42dbb7fe0, 0x7fad6035c8e0, 0xc42d96d380, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d96d380, 0xa934c0, 0xc42d96d2c0, 0x7fad6035c8e0, 0xc42d96d380, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d96d380, 0xa934c0, 0xc42d96d2c0, 0xc42dbb7fc0, 0xc42dc4e000, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xffffffff, 0x0, 0x19, 0xc42d7117e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xc42dbb7fc0, 0xc42dc4e020, 0xc42d711878, 0x558012, 0x588a5743, 0x2a4230cc)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xc42d711900, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xc42dbb7fc0, 0xc42dc4e040, 0xa91b40, 0xc42dc4e060, 0xc42dbb7fc0, 0xc42dc4e060)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xc420246100, 0xc42d96b770, 0xc42d711a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d96d1a0, 0xa934c0, 0xc42d96d2c0, 0xc42dbb7fc0, 0xc42dbb7fe0, 0x4e, 0xc42580fb18, 0x0, 0xc42580fb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d96d1a0, 0xc42d5aa780, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d055300, 0xc420331380, 0x10265d8, 0xc42d96d260, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d055300, 0xc42d96d260)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d055300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8354 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42cbe4810, 0xc42cbe4840, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42cbe4810, 0xc43ddc4e00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42cbe4810, 0xc4379f5e60, 0x0, 0xc42cb08288, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42cbe4810, 0xa74a80, 0xc4379f5e60, 0x7fad6035c8e0, 0xc42cbe4810, 0x1038500, 0xc42cbe47e0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42cbe47e0, 0xa74a80, 0xc4379f5e60, 0xc43ddc4d00, 0xc43ddc4d20, 0x7fad6035c8e0, 0xc42cbe47e0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cbe47e0, 0xa74a80, 0xc4379f5e60, 0x7fad6035c8e0, 0xc42cbe47e0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42cbe47e0, 0xa74a80, 0xc4379f5e60, 0xc43ddc4d00, 0xc43ddc4d40, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xffffffff, 0x0, 0x19, 0xc439edb7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xc43ddc4d00, 0xc43ddc4d60, 0xc439edb878, 0x558012, 0x588a579b, 0x3850790e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xed01c4e9b, 0x0, 0x0, 0xd358d)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xc43ddc4d00, 0xc43ddc4d80, 0xa91b40, 0xc43ddc4da0, 0xc43ddc4d00, 0xc43ddc4da0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xc420246100, 0xc42ce488f0, 0xc439edba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42ce53e90, 0xa74a80, 0xc4379f5e60, 0xc43ddc4d00, 0xc43ddc4d20, 0x9bf6, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42ce53e90, 0xc42ba4dbc0, 0xc420179d60, 0x0, 0x0, 0x637573070a0f0a63, 0x7274041273736563)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad300, 0xc420331380, 0x10265c0, 0xc42ce53ec0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad300, 0xc42ce53ec0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42cbad300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9896 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4387addec)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc4387adde0)
	/usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc4387adde0, 0xc44346c7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e

goroutine 956 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048eb04)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048eb00)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x2f2591367954edce)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc429d49e30, 0xc420162418, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc429d491d0, 0xc429d49e30, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc429d491d0, 0xc4257999c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc429d491d0, 0xc4258dc560, 0x0, 0xc429d4eed8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc429d491d0, 0xa74a80, 0xc4258dc560, 0x7fad6035c8e0, 0xc429d491d0, 0x1038500, 0xc429d491a0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc429d491a0, 0xa74a80, 0xc4258dc560, 0xc425798e20, 0xc425798e80, 0x7fad6035c8e0, 0xc429d491a0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429d491a0, 0xa74a80, 0xc4258dc560, 0x7fad6035c8e0, 0xc429d491a0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc429d491a0, 0xa74a80, 0xc4258dc560, 0xc425798e20, 0xc425798ec0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xffffffff, 0x0, 0x19, 0xc4294737e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xc425798e20, 0xc425798ee0, 0xc429473878, 0x558012, 0x588a5741, 0x32ff0d58)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xed01c4e41, 0x0, 0x0, 0x126687)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xc425798e20, 0xc425798f00, 0xa91b40, 0xc425798f20, 0xc425798e20, 0xc425798f20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xc420246100, 0xc429cb7770, 0xc429473a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc429ccc150, 0xa74a80, 0xc4258dc560, 0xc425798e20, 0xc425798e80, 0xa86c, 0xc42023db18, 0x0, 0xc42023db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc429ccc150, 0xc427a01e60, 0xc420179d60, 0x0, 0x0, 0x312d70691c126564, 0x2d312d30322d3237)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed0800, 0xc420331380, 0x10265c0, 0xc429ccc180, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed0800, 0xc429ccc180)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed0800)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8565 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44293d9e0, 0xc44293da10, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44293d9e0, 0xc43fabcb20, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44293d9e0, 0xc43fab6a20, 0x0, 0xc429cfde08, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44293d9e0, 0xa74a80, 0xc43fab6a20, 0x7fad6035c8e0, 0xc44293d9e0, 0x1038500, 0xc44293d9b0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44293d9b0, 0xa74a80, 0xc43fab6a20, 0xc43fabc960, 0xc43fabca40, 0x7fad6035c8e0, 0xc44293d9b0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44293d9b0, 0xa74a80, 0xc43fab6a20, 0x7fad6035c8e0, 0xc44293d9b0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44293d9b0, 0xa74a80, 0xc43fab6a20, 0xc43fabc960, 0xc43fabca60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xffffffff, 0x0, 0x19, 0xc438c977e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xc43fabc960, 0xc43fabca80, 0xc438c97878, 0x558012, 0x588a59b7, 0x379b16b1)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xed01c50b7, 0x0, 0x0, 0x3bb05)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xc43fabc960, 0xc43fabcaa0, 0xa91b40, 0xc43fabcac0, 0xc43fabc960, 0xc43fabcac0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xc420246100, 0xc429d26b50, 0xc438c97a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44293cf30, 0xa74a80, 0xc43fab6a20, 0xc43fabc960, 0xc43fabca40, 0x1571, 0xc420e1db18, 0x0, 0xc420e1db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44293cf30, 0xc43ead84e0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc433f6ee00, 0xc420331380, 0x10265c0, 0xc44293cf60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc433f6ee00, 0xc44293cf60)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc433f6ee00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8550 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc422c45b60, 0xc422c45b90, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc422c45b60, 0xc433bd8b40, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc422c45b60, 0xc433c797c0, 0x0, 0xc422b51c28, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc422c45b60, 0xa74a80, 0xc433c797c0, 0x7fad6035c8e0, 0xc422c45b60, 0x1038500, 0xc422c45b30)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc422c45b30, 0xa74a80, 0xc433c797c0, 0xc433bd89c0, 0xc433bd89e0, 0x7fad6035c8e0, 0xc422c45b30, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc422c45b30, 0xa74a80, 0xc433c797c0, 0x7fad6035c8e0, 0xc422c45b30, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc422c45b30, 0xa74a80, 0xc433c797c0, 0xc433bd89c0, 0xc433bd8a00, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xffffffff, 0x0, 0x19, 0xc448c337e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xc433bd89c0, 0xc433bd8a20, 0xc448c33878, 0x558012, 0x588a57b9, 0x390dce08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xed01c4eb9, 0x0, 0x0, 0x45cf1a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xc433bd89c0, 0xc433bd8a40, 0xa91b40, 0xc433bd8aa0, 0xc433bd89c0, 0xc433bd8aa0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xc420246100, 0xc422165600, 0xc448c33a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4241a7a70, 0xa74a80, 0xc433c797c0, 0xc433bd89c0, 0xc433bd89e0, 0xbb32, 0xc420047b18, 0x0, 0xc420047b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4241a7a70, 0xc4235d56e0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f006322)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc425cf3c00, 0xc420331380, 0x10265c0, 0xc4241a7aa0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc425cf3c00, 0xc4241a7aa0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc425cf3c00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9963 [chan receive, 8 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc446610870, 0xc435e3bc20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc446610870, 0xc435e3bc20)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc446610870, 0xc435e3bc20)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc4417fa480, 0xc435e3bc20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc4417fa480, 0xc435e3bc20)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc4417fa480, 0xc435e3bc20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc4417fa440, 0xc435e3ba40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc4417fa440, 0xc435e3ba40)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc4417fa420, 0xc435e3ba40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc4417fa420, 0xc435e3ba40)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc42bd16b60, 0xc435e3ba40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc42bd16b60, 0xc435e3b950)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc42bd16b60, 0xc435e3b950)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc43269b480, 0x1038440, 0xc43ed8f1c0)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 5310 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048f704)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048f700)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x2aa3be3b56d99d3e)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc448cff200, 0xc420162418, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc448cfe9c0, 0xc448cff200, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc448cfe9c0, 0xc42b93a900, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc448cfe9c0, 0xc42b9994a0, 0x0, 0xc448c5e650, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc448cfe9c0, 0xa74a80, 0xc42b9994a0, 0x7fad6035c8e0, 0xc448cfe9c0, 0x1038500, 0xc448cfe990)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc448cfe990, 0xa74a80, 0xc42b9994a0, 0xc42b93fd20, 0xc42b93fd60, 0x7fad6035c8e0, 0xc448cfe990, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc448cfe990, 0xa74a80, 0xc42b9994a0, 0x7fad6035c8e0, 0xc448cfe990, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc448cfe990, 0xa74a80, 0xc42b9994a0, 0xc42b93fd20, 0xc42b93fda0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xffffffff, 0x0, 0x19, 0xc43ee3d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xc42b93fd20, 0xc42b93fdc0, 0xc43ee3d878, 0x558012, 0x588a575f, 0x32fe05f8)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xed01c4e5f, 0x0, 0x0, 0x15eca2)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xc42b93fd20, 0xc42b93fde0, 0xa91b40, 0xc42b93fe20, 0xc42b93fd20, 0xc42b93fe20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xc420246100, 0xc4493beae0, 0xc43ee3da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4493ca6c0, 0xa74a80, 0xc42b9994a0, 0xc42b93fd20, 0xc42b93fd60, 0xae8b, 0xc42023db18, 0x0, 0xc42023db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4493ca6c0, 0xc43eb0d560, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421003200, 0xc420331380, 0x10265c0, 0xc4493ca6f0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421003200, 0xc4493ca6f0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421003200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8149 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d4f39b0, 0xc42d4f39e0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d4f39b0, 0xc437acb8c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d4f39b0, 0xc437af3980, 0x0, 0xc42d4e08c0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d4f39b0, 0xa74a80, 0xc437af3980, 0x7fad6035c8e0, 0xc42d4f39b0, 0x1038500, 0xc42d4f3980)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d4f3980, 0xa74a80, 0xc437af3980, 0xc437acb780, 0xc437acb7a0, 0x7fad6035c8e0, 0xc42d4f3980, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d4f3980, 0xa74a80, 0xc437af3980, 0x7fad6035c8e0, 0xc42d4f3980, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d4f3980, 0xa74a80, 0xc437af3980, 0xc437acb780, 0xc437acb7e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xffffffff, 0x0, 0x19, 0xc439edd7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xc437acb780, 0xc437acb800, 0xc439edd878, 0x558012, 0x588a579b, 0x36ace23c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xc439edd900, 0x0, 0x0, 0xbf)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xc437acb780, 0xc437acb820, 0xa91b40, 0xc437acb860, 0xc437acb780, 0xc437acb860)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xc420246100, 0xc427518680, 0xc439edda70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc428ee6120, 0xa74a80, 0xc437af3980, 0xc437acb780, 0xc437acb7a0, 0xa5f8, 0xc42023cb18, 0x0, 0xc42023cb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc428ee6120, 0xc44088f2c0, 0xc420179d60, 0x0, 0x0, 0x6c616e7265746e69, 0x412656c020a0a0a)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c300, 0xc420331380, 0x10265c0, 0xc428ee61e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c300, 0xc428ee61e0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 11110 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a
github.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc43de094a0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc434eb1b10, 0xc43de094a0, 0x10343c0, 0xc4201e3040)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326

goroutine 8324 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42e393d10, 0xc42e393d40, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42e393d10, 0xc437c30680, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42e393d10, 0xc437cbb800, 0x0, 0xc42a5bac10, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42e393d10, 0xa74a80, 0xc437cbb800, 0x7fad6035c8e0, 0xc42e393d10, 0x1038500, 0xc42e393ce0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42e393ce0, 0xa74a80, 0xc437cbb800, 0xc437c30540, 0xc437c30560, 0x7fad6035c8e0, 0xc42e393ce0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e393ce0, 0xa74a80, 0xc437cbb800, 0x7fad6035c8e0, 0xc42e393ce0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42e393ce0, 0xa74a80, 0xc437cbb800, 0xc437c30540, 0xc437c30580, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0xffffffff, 0x0, 0x19, 0xc4464b37e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0xc437c30540, 0xc437c305a0, 0xc4464b3878, 0x558012, 0x588a5798, 0xdc5a87c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0x411715, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0xc437c30540, 0xc437c305c0, 0xa91b40, 0xc437c30600, 0xc437c30540, 0xc437c30600)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0xc420246100, 0xc42a685c90, 0xc4464b3a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4248f1b90, 0xa74a80, 0xc437cbb800, 0xc437c30540, 0xc437c30560, 0x6b39, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4248f1b90, 0xc42ba4c120, 0xc420179d60, 0x0, 0x0, 0x646e696b040a100a, 0x61646174656d0812)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3200, 0xc420331380, 0x10265c0, 0xc4267f4030, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3200, 0xc4267f4030)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8551 [chan receive, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc42443cd20, 0xc443d482d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc42443cd20, 0xc443d482d0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc42443cd20, 0xc443d482d0)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc433a8a760, 0xc443d482d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc433a8a760, 0xc443d482d0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc433a8a760, 0xc443d482d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc433a8a740, 0xc443d480f0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc433a8a740, 0xc443d480f0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc433a8a6e0, 0xc443d480f0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc433a8a6e0, 0xc443d480f0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4292dfba0, 0xc443d480f0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4292dfba0, 0xc443d48000)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4292dfba0, 0xc443d48000)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc43a2a0a80, 0x1038440, 0xc427f107c0)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 8224 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42e2649c0, 0xc42e2649f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42e2649c0, 0xc437c38700, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42e2649c0, 0xc437c85c40, 0x0, 0xc42a5b8838, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42e2649c0, 0xa74a80, 0xc437c85c40, 0x7fad6035c8e0, 0xc42e2649c0, 0x1038500, 0xc42e264990)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42e264990, 0xa74a80, 0xc437c85c40, 0xc437c38580, 0xc437c38600, 0x7fad6035c8e0, 0xc42e264990, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e264990, 0xa74a80, 0xc437c85c40, 0x7fad6035c8e0, 0xc42e264990, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42e264990, 0xa74a80, 0xc437c85c40, 0xc437c38580, 0xc437c38620, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xffffffff, 0x0, 0x19, 0xc4454c97e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xc437c38580, 0xc437c38640, 0xc4454c9878, 0x558012, 0x588a5798, 0xdc3489f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xed01c4e98, 0x0, 0x0, 0x172211)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xc437c38580, 0xc437c38660, 0xa91b40, 0xc437c38680, 0xc437c38580, 0xc437c38680)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xc420246100, 0xc42bc95f40, 0xc4454c9a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc426b9e150, 0xa74a80, 0xc437c85c40, 0xc437c38580, 0xc437c38600, 0x4809, 0xc420e1db18, 0x0, 0xc420e1db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc426b9e150, 0xc4296faea0, 0xc420179d60, 0x0, 0x0, 0xc43ceb47a8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4275e7c00, 0xc420331380, 0x10265c0, 0xc426b9e180, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4275e7c00, 0xc426b9e180)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4275e7c00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9093 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc443740930, 0xc4437409f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc443740930, 0xc43cae2ec0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc443740930, 0xc438b40d20, 0x0, 0xc447b12208, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc443740930, 0xa74a80, 0xc438b40d20, 0x7fad6035c8e0, 0xc443740930, 0x1038500, 0xc4437408d0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4437408d0, 0xa74a80, 0xc438b40d20, 0xc43d181fa0, 0xc43cae2080, 0x7fad6035c8e0, 0xc4437408d0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4437408d0, 0xa74a80, 0xc438b40d20, 0x7fad6035c8e0, 0xc4437408d0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4437408d0, 0xa74a80, 0xc438b40d20, 0xc43d181fa0, 0xc43cae2280, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xffffffff, 0x0, 0x19, 0xc444c017e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xc43d181fa0, 0xc43cae2540, 0xc444c01878, 0x558012, 0x588a5810, 0xbca599a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xc444c01900, 0x0, 0x0, 0x4d)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xc43d181fa0, 0xc43cae2760, 0xa91b40, 0xc43cae29a0, 0xc43d181fa0, 0xc43cae29a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xc420246100, 0xc43e19f190, 0xc444c01a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc444f08630, 0xa74a80, 0xc438b40d20, 0xc43d181fa0, 0xc43cae2080, 0x7e56, 0xc42023bb18, 0x0, 0xc42023bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc444f08630, 0xc449d8a420, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a6d00, 0xc420331380, 0x10265c0, 0xc444f08870, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a6d00, 0xc444f08870)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a6d00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8564 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43c8cbcb0, 0xc43c8cbce0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43c8cbcb0, 0xc43bba7d20, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43c8cbcb0, 0xc44092a560, 0x0, 0xc4272a5c18, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43c8cbcb0, 0xa74a80, 0xc44092a560, 0x7fad6035c8e0, 0xc43c8cbcb0, 0x1038500, 0xc43c8cbc80)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43c8cbc80, 0xa74a80, 0xc44092a560, 0xc43bba7c20, 0xc43bba7c40, 0x7fad6035c8e0, 0xc43c8cbc80, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c8cbc80, 0xa74a80, 0xc44092a560, 0x7fad6035c8e0, 0xc43c8cbc80, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43c8cbc80, 0xa74a80, 0xc44092a560, 0xc43bba7c20, 0xc43bba7c60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xffffffff, 0x0, 0x19, 0xc438c957e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xc43bba7c20, 0xc43bba7c80, 0xc438c95878, 0x558012, 0x588a59b7, 0x3416450e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xed01c50b7, 0x0, 0x0, 0x1559c1)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xc43bba7c20, 0xc43bba7ca0, 0xa91b40, 0xc43bba7cc0, 0xc43bba7c20, 0xc43bba7cc0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xc420246100, 0xc42d842630, 0xc438c95a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43c14bf50, 0xa74a80, 0xc44092a560, 0xc43bba7c20, 0xc43bba7c40, 0x4558, 0xc420e1db18, 0x0, 0xc420e1db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43c14bf50, 0xc43ead8300, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6eb00, 0xc420331380, 0x10265c0, 0xc43c14bf80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6eb00, 0xc43c14bf80)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6eb00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 11987 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d1ba570, 0xc42d1ba630, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1ba570, 0xc44320c400, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1ba570, 0xc444b165a0, 0x0, 0xc42fe8a450, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d1ba570, 0xa74a80, 0xc444b165a0, 0x7fad6035c8e0, 0xc42d1ba570, 0x1038500, 0xc42d1ba4b0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d1ba4b0, 0xa74a80, 0xc444b165a0, 0xc44320c300, 0xc44320c320, 0x7fad6035c8e0, 0xc42d1ba4b0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1ba4b0, 0xa74a80, 0xc444b165a0, 0x7fad6035c8e0, 0xc42d1ba4b0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d1ba4b0, 0xa74a80, 0xc444b165a0, 0xc44320c300, 0xc44320c340, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0xffffffff, 0x0, 0x19, 0xc4377097e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0xc44320c300, 0xc44320c360, 0xc437709878, 0x558012, 0x588a59d5, 0x3832a21c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0x41167e, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0xc44320c300, 0xc44320c380, 0xa91b40, 0xc44320c3a0, 0xc44320c300, 0xc44320c3a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0xc420246100, 0xc430f2b1f0, 0xc437709a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d3a5260, 0xa74a80, 0xc444b165a0, 0xc44320c300, 0xc44320c320, 0x4a1b, 0xc440d64b18, 0x0, 0xc440d64b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d3a5260, 0xc44103c960, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2100, 0xc420331380, 0x10265c0, 0xc42d3a5f20, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2100, 0xc42d3a5f20)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2100)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8863 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc449db7cb0, 0xc449db7ce0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc449db7cb0, 0xc44a5053e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc449db7cb0, 0xc43e3324c0, 0x0, 0xc445048050, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc449db7cb0, 0xa74a80, 0xc43e3324c0, 0x7fad6035c8e0, 0xc449db7cb0, 0x1038500, 0xc449db7c80)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc449db7c80, 0xa74a80, 0xc43e3324c0, 0xc44a5052e0, 0xc44a505300, 0x7fad6035c8e0, 0xc449db7c80, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449db7c80, 0xa74a80, 0xc43e3324c0, 0x7fad6035c8e0, 0xc449db7c80, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc449db7c80, 0xa74a80, 0xc43e3324c0, 0xc44a5052e0, 0xc44a505320, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xffffffff, 0x0, 0x19, 0xc43ed837e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xc44a5052e0, 0xc44a505340, 0xc43ed83878, 0x558012, 0x588a57f5, 0x371d60e2)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xed01c4ef5, 0x0, 0x0, 0xafff3)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xc44a5052e0, 0xc44a505360, 0xa91b40, 0xc44a505380, 0xc44a5052e0, 0xc44a505380)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xc420246100, 0xc426faf720, 0xc43ed83a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43ac0fd70, 0xa74a80, 0xc43e3324c0, 0xc44a5052e0, 0xc44a505300, 0x4ef8, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43ac0fd70, 0xc42c213aa0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f1a9759)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87c00, 0xc420331380, 0x10265c0, 0xc43ac0fda0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87c00, 0xc43ac0fda0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87c00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8439 [chan send, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc42c050d20, 0xc429a59a40, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 8328 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d30f5f0, 0xc42d30f620, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d30f5f0, 0xc437a69900, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d30f5f0, 0xc437c31820, 0x0, 0xc42d311020, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d30f5f0, 0xa74a80, 0xc437c31820, 0x7fad6035c8e0, 0xc42d30f5f0, 0x1038500, 0xc42d30f5c0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d30f5c0, 0xa74a80, 0xc437c31820, 0xc437a697e0, 0xc437a69820, 0x7fad6035c8e0, 0xc42d30f5c0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d30f5c0, 0xa74a80, 0xc437c31820, 0x7fad6035c8e0, 0xc42d30f5c0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d30f5c0, 0xa74a80, 0xc437c31820, 0xc437a697e0, 0xc437a69840, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xffffffff, 0x0, 0x19, 0xc4464b57e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xc437a697e0, 0xc437a69860, 0xc4464b5878, 0x558012, 0x588a579b, 0x36b5bea6)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xed01c4e9b, 0x0, 0x0, 0xe9a86)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xc437a697e0, 0xc437a69880, 0xa91b40, 0xc437a698a0, 0xc437a697e0, 0xc437a698a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xc420246100, 0xc42a5bacd0, 0xc4464b5a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc426bbc0f0, 0xa74a80, 0xc437c31820, 0xc437a697e0, 0xc437a69820, 0xa56e, 0xc420049b18, 0x0, 0xc420049b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc426bbc0f0, 0xc42ba4c960, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516500, 0xc420331380, 0x10265c0, 0xc426bbc120, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516500, 0xc426bbc120)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516500)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8072 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4244583c0, 0xc4244583f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4244583c0, 0xc433ff26a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4244583c0, 0xc4340f8140, 0x0, 0xc425ba8a88, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4244583c0, 0xa74a80, 0xc4340f8140, 0x7fad6035c8e0, 0xc4244583c0, 0x1038500, 0xc424458390)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc424458390, 0xa74a80, 0xc4340f8140, 0xc433ff2540, 0xc433ff2560, 0x7fad6035c8e0, 0xc424458390, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc424458390, 0xa74a80, 0xc4340f8140, 0x7fad6035c8e0, 0xc424458390, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc424458390, 0xa74a80, 0xc4340f8140, 0xc433ff2540, 0xc433ff2580, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xffffffff, 0x0, 0x19, 0xc43b1b57e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xc433ff2540, 0xc433ff25a0, 0xc43b1b5878, 0x558012, 0x588a57b6, 0xbf5352e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xed01c4eb6, 0x0, 0x0, 0xe71e4)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xc433ff2540, 0xc433ff25e0, 0xa91b40, 0xc433ff2620, 0xc433ff2540, 0xc433ff2620)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xc420246100, 0xc4278f60a0, 0xc43b1b5a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc427c9a210, 0xa74a80, 0xc4340f8140, 0xc433ff2540, 0xc433ff2560, 0x73d9, 0xc42023cb18, 0x0, 0xc42023cb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc427c9a210, 0xc42779e0c0, 0xc420179d60, 0x0, 0x0, 0x4d1279726f746973, 0x6f642e7865646e69)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bc000, 0xc420331380, 0x10265c0, 0xc427c9a240, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bc000, 0xc427c9a240)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bc000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10350 [chan send, 8 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc43aa882d0, 0xc427521aa0, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 11427 [chan receive, 4 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc437232960, 0xc441ff1b30)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc437232960, 0xc441ff1b30)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc437232960, 0xc441ff1b30)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc43d66fc80, 0xc441ff1b30)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc43d66fc80, 0xc441ff1b30)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc43d66fc80, 0xc441ff1b30)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc43d66fc40, 0xc441ff1950)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc43d66fc40, 0xc441ff1950)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc43d66fc20, 0xc441ff1950)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc43d66fc20, 0xc441ff1950)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc42bd16270, 0xc441ff1950)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc42bd16270, 0xc441ff1860)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc42bd16270, 0xc441ff1860)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc42ce81e80, 0x1038440, 0xc4391235c0)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 11217 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42e09da10, 0xc42e09da70, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42e09da10, 0xc44448ba60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42e09da10, 0xc43a130000, 0x0, 0xc4351d5b50, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42e09da10, 0xa74a80, 0xc43a130000, 0x7fad6035c8e0, 0xc42e09da10, 0x1038500, 0xc42e09d9b0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42e09d9b0, 0xa74a80, 0xc43a130000, 0xc44448b960, 0xc44448b980, 0x7fad6035c8e0, 0xc42e09d9b0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e09d9b0, 0xa74a80, 0xc43a130000, 0x7fad6035c8e0, 0xc42e09d9b0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42e09d9b0, 0xa74a80, 0xc43a130000, 0xc44448b960, 0xc44448b9a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xffffffff, 0x0, 0x19, 0xc4491b17e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xc44448b960, 0xc44448b9c0, 0xc4491b1878, 0x558012, 0x588a595d, 0x3aab877f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xed01c505d, 0x0, 0x0, 0xbac3b)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xc44448b960, 0xc44448b9e0, 0xa91b40, 0xc44448ba00, 0xc44448b960, 0xc44448ba00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xc420246100, 0xc435236050, 0xc4491b1a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42e4228a0, 0xa74a80, 0xc43a130000, 0xc44448b960, 0xc44448b980, 0x74a2, 0xc440d64b18, 0x0, 0xc440d64b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42e4228a0, 0xc448a02000, 0xc420179d60, 0x0, 0x0, 0xc434e02fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a7d00, 0xc420331380, 0x10265c0, 0xc42e422900, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a7d00, 0xc42e422900)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a7d00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8701 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc431022600, 0xc431022630, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc431022600, 0xc4327302a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc431022600, 0xc4328d87c0, 0x0, 0xc4310247d8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc431022600, 0xa74a80, 0xc4328d87c0, 0x7fad6035c8e0, 0xc431022600, 0x1038500, 0xc4310225d0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4310225d0, 0xa74a80, 0xc4328d87c0, 0xc432730100, 0xc432730120, 0x7fad6035c8e0, 0xc4310225d0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4310225d0, 0xa74a80, 0xc4328d87c0, 0x7fad6035c8e0, 0xc4310225d0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4310225d0, 0xa74a80, 0xc4328d87c0, 0xc432730100, 0xc432730180, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0xffffffff, 0x0, 0x19, 0xc42eba97e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0xc432730100, 0xc4327301a0, 0xc42eba9878, 0x558012, 0x588a57d7, 0x33ad55a1)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0x411715, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0xc432730100, 0xc4327301c0, 0xa91b40, 0xc432730220, 0xc432730100, 0xc432730220)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0xc420246100, 0xc4421e7910, 0xc42eba9a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43acec270, 0xa74a80, 0xc4328d87c0, 0xc432730100, 0xc432730120, 0x7a5e, 0xc420313b18, 0x0, 0xc420313b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43acec270, 0xc441296900, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f69cfe7)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1a00, 0xc420331380, 0x10265c0, 0xc43acec2a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1a00, 0xc43acec2a0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc421ed1a00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8566 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc440460690, 0xc4404606c0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc440460690, 0xc448f51aa0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc440460690, 0xc448f30b80, 0x0, 0xc42e5dd498, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc440460690, 0xa74a80, 0xc448f30b80, 0x7fad6035c8e0, 0xc440460690, 0x1038500, 0xc440460660)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc440460660, 0xa74a80, 0xc448f30b80, 0xc448f519a0, 0xc448f519c0, 0x7fad6035c8e0, 0xc440460660, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440460660, 0xa74a80, 0xc448f30b80, 0x7fad6035c8e0, 0xc440460660, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc440460660, 0xa74a80, 0xc448f30b80, 0xc448f519a0, 0xc448f519e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xffffffff, 0x0, 0x19, 0xc438c937e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xc448f519a0, 0xc448f51a00, 0xc438c93878, 0x558012, 0x588a59b7, 0x37e74cb5)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xed01c50b7, 0x0, 0x0, 0x92d84)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xc448f519a0, 0xc448f51a20, 0xa91b40, 0xc448f51a40, 0xc448f519a0, 0xc448f51a40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xc420246100, 0xc4288a5000, 0xc438c93a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4461b4750, 0xa74a80, 0xc448f30b80, 0xc448f519a0, 0xc448f519c0, 0x6564, 0xc420314b18, 0x0, 0xc420314b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4461b4750, 0xc43ead8780, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f300, 0xc420331380, 0x10265c0, 0xc4461b4780, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f300, 0xc4461b4780)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8146 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc447f6ddd0, 0xc447f6de00, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc447f6ddd0, 0xc438caf440, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc447f6ddd0, 0xc438d08060, 0x0, 0xc4443e8b00, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc447f6ddd0, 0xa74a80, 0xc438d08060, 0x7fad6035c8e0, 0xc447f6ddd0, 0x1038500, 0xc447f6dda0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc447f6dda0, 0xa74a80, 0xc438d08060, 0xc438caf320, 0xc438caf340, 0x7fad6035c8e0, 0xc447f6dda0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447f6dda0, 0xa74a80, 0xc438d08060, 0x7fad6035c8e0, 0xc447f6dda0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc447f6dda0, 0xa74a80, 0xc438d08060, 0xc438caf320, 0xc438caf360, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xffffffff, 0x0, 0x19, 0xc42d1f37e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xc438caf320, 0xc438caf380, 0xc42d1f3878, 0x558012, 0x588a577d, 0x33f8d325)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xed01c4e7d, 0x0, 0x0, 0x186123)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xc438caf320, 0xc438caf3a0, 0xa91b40, 0xc438caf3c0, 0xc438caf320, 0xc438caf3c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xc420246100, 0xc442adb0e0, 0xc42d1f3a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442346330, 0xa74a80, 0xc438d08060, 0xc438caf320, 0xc438caf340, 0x8687, 0xc420044b18, 0x0, 0xc420044b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442346330, 0xc423b1fec0, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abe00, 0xc420331380, 0x10265c0, 0xc442346360, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abe00, 0xc442346360)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abe00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8987 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43f7318c0, 0xc43f7318f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43f7318c0, 0xc4481f6a00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43f7318c0, 0xc43dd56d00, 0x0, 0xc43fc7cdd0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43f7318c0, 0xa74a80, 0xc43dd56d00, 0x7fad6035c8e0, 0xc43f7318c0, 0x1038500, 0xc43f731890)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43f731890, 0xa74a80, 0xc43dd56d00, 0xc4481f68e0, 0xc4481f6900, 0x7fad6035c8e0, 0xc43f731890, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43f731890, 0xa74a80, 0xc43dd56d00, 0x7fad6035c8e0, 0xc43f731890, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43f731890, 0xa74a80, 0xc43dd56d00, 0xc4481f68e0, 0xc4481f6940, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xffffffff, 0x0, 0x19, 0xc44a48b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xc4481f68e0, 0xc4481f6960, 0xc44a48b878, 0x558012, 0x588a5813, 0x36eaff71)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xed01c4f13, 0x0, 0x0, 0x669e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xc4481f68e0, 0xc4481f6980, 0xa91b40, 0xc4481f69a0, 0xc4481f68e0, 0xc4481f69a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xc420246100, 0xc442d71730, 0xc44a48ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc441afaed0, 0xa74a80, 0xc43dd56d00, 0xc4481f68e0, 0xc4481f6900, 0x34ca, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc441afaed0, 0xc4431f7a40, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1a00, 0xc420331380, 0x10265c0, 0xc441afb2f0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1a00, 0xc441afb2f0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1a00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8080 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc423781020, 0xc423781050, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc423781020, 0xc433d34460, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc423781020, 0xc433d80b80, 0x0, 0xc423602a10, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc423781020, 0xa74a80, 0xc433d80b80, 0x7fad6035c8e0, 0xc423781020, 0x1038500, 0xc423780ff0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc423780ff0, 0xa74a80, 0xc433d80b80, 0xc433d34280, 0xc433d342c0, 0x7fad6035c8e0, 0xc423780ff0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423780ff0, 0xa74a80, 0xc433d80b80, 0x7fad6035c8e0, 0xc423780ff0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc423780ff0, 0xa74a80, 0xc433d80b80, 0xc433d34280, 0xc433d342e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xffffffff, 0x0, 0x19, 0xc448f017e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xc433d34280, 0xc433d34320, 0xc448f01878, 0x558012, 0x588a57b9, 0x336997c9)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xed01c4eb9, 0x0, 0x0, 0xf35b5)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xc433d34280, 0xc433d34340, 0xa91b40, 0xc433d34380, 0xc433d34280, 0xc433d34380)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xc420246100, 0xc423b4dec0, 0xc448f01a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4239e4a20, 0xa74a80, 0xc433d80b80, 0xc433d34280, 0xc433d342c0, 0x7594, 0xc420319b18, 0x0, 0xc420319b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4239e4a20, 0xc42779f8c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5d1149)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bcd00, 0xc420331380, 0x10265c0, 0xc4239e4a50, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bcd00, 0xc4239e4a50)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bcd00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 11290 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc429a53200, 0xc429a53320, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc429a53200, 0xc440d0f2a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc429a53200, 0xc43c0a5340, 0x0, 0xc432dd5cc0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc429a53200, 0xa74a80, 0xc43c0a5340, 0x7fad6035c8e0, 0xc429a53200, 0x1038500, 0xc429a53140)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc429a53140, 0xa74a80, 0xc43c0a5340, 0xc440d0f180, 0xc440d0f1a0, 0x7fad6035c8e0, 0xc429a53140, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429a53140, 0xa74a80, 0xc43c0a5340, 0x7fad6035c8e0, 0xc429a53140, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc429a53140, 0xa74a80, 0xc43c0a5340, 0xc440d0f180, 0xc440d0f1c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xffffffff, 0x0, 0x19, 0xc437c817e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xc440d0f180, 0xc440d0f1e0, 0xc437c81878, 0x558012, 0x588a5978, 0x12bd2347)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xed01c5078, 0x0, 0x0, 0xdd0ec0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xc440d0f180, 0xc440d0f220, 0xa91b40, 0xc440d0f240, 0xc440d0f180, 0xc440d0f240)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xc420246100, 0xc432fa2cf0, 0xc437c81a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a4a0e10, 0xa74a80, 0xc43c0a5340, 0xc440d0f180, 0xc440d0f1a0, 0x7885, 0xc420e18b18, 0x0, 0xc420e18b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a4a0e10, 0xc442bd4360, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa000, 0xc420331380, 0x10265c0, 0xc42a4a0fc0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa000, 0xc42a4a0fc0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 1207 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204962c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204962c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42ca1eb80, 0x3, 0x3, 0xc429471448, 0x0, 0xc42c9c1800)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42c9cf7a0, 0x159dc604330, 0x159dc64d710, 0xc42ca1eb80, 0x3, 0x3, 0xc42c9cf700, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42c9cf7a0, 0xc42c9cf680, 0x0, 0xc42c9cc4d8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42c9cf7a0, 0xa934c0, 0xc42c9cf680, 0x7fad6035c8e0, 0xc42c9cf7a0, 0x1038500, 0xc42c9cf770)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c9cf770, 0xa934c0, 0xc42c9cf680, 0xc42ca1ea80, 0xc42ca1eaa0, 0x7fad6035c8e0, 0xc42c9cf770, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9cf770, 0xa934c0, 0xc42c9cf680, 0x7fad6035c8e0, 0xc42c9cf770, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c9cf770, 0xa934c0, 0xc42c9cf680, 0xc42ca1ea80, 0xc42ca1eac0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xffffffff, 0x0, 0x19, 0xc4294717e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xc42ca1ea80, 0xc42ca1eae0, 0xc429471878, 0x558012, 0x588a5743, 0x853622e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xed01c4e43, 0x0, 0x0, 0xae05)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xc42ca1ea80, 0xc42ca1eb00, 0xa91b40, 0xc42ca1eb20, 0xc42ca1ea80, 0xc42ca1eb20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xc420246100, 0xc42c9cc480, 0xc429471a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42c9cf560, 0xa934c0, 0xc42c9cf680, 0xc42ca1ea80, 0xc42ca1eaa0, 0x65, 0xc420319b18, 0x0, 0xc420319b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42c9cf560, 0xc42ba39140, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429599c00, 0xc420331380, 0x10265d8, 0xc42c9cf620, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429599c00, 0xc42c9cf620)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc429599c00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 12194 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43a487560, 0xc43a487590, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43a487560, 0xc43d7009e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43a487560, 0xc43e05fec0, 0x0, 0xc426d06b28, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43a487560, 0xa74a80, 0xc43e05fec0, 0x7fad6035c8e0, 0xc43a487560, 0x1038500, 0xc43a487530)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43a487530, 0xa74a80, 0xc43e05fec0, 0xc43d7008e0, 0xc43d700900, 0x7fad6035c8e0, 0xc43a487530, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43a487530, 0xa74a80, 0xc43e05fec0, 0x7fad6035c8e0, 0xc43a487530, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43a487530, 0xa74a80, 0xc43e05fec0, 0xc43d7008e0, 0xc43d700920, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0xffffffff, 0x0, 0x19, 0xc435d4b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0xc43d7008e0, 0xc43d700940, 0xc435d4b878, 0x558012, 0x588a59f3, 0x3b313cc2)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0x41167e, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0xc43d7008e0, 0xc43d700960, 0xa91b40, 0xc43d700980, 0xc43d7008e0, 0xc43d700980)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0xc420246100, 0xc427760030, 0xc435d4ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc449610a80, 0xa74a80, 0xc43e05fec0, 0xc43d7008e0, 0xc43d700900, 0x52ec, 0xc435e50b18, 0x0, 0xc435e50b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc449610a80, 0xc449598de0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065100, 0xc420331380, 0x10265c0, 0xc449610ab0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065100, 0xc449610ab0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065100)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 1233 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048bb84)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048bb80)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x476db0d6341b0a2)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d5080a0, 0x2, 0x2, 0xc429477448, 0x0, 0xc42d4f0200)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d506240, 0x159dc604330, 0x159dc64d710, 0xc42d5080a0, 0x2, 0x2, 0xc42d506200, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d506240, 0xc42d506150, 0x0, 0xc42d508078, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d506240, 0xa934c0, 0xc42d506150, 0x7fad6035c8e0, 0xc42d506240, 0x1038500, 0xc42d506210)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d506210, 0xa934c0, 0xc42d506150, 0xc42d4ee4e0, 0xc42d4ee500, 0x7fad6035c8e0, 0xc42d506210, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d506210, 0xa934c0, 0xc42d506150, 0x7fad6035c8e0, 0xc42d506210, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d506210, 0xa934c0, 0xc42d506150, 0xc42d4ee4e0, 0xc42d4ee520, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xffffffff, 0x0, 0x19, 0xc4294777e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xc42d4ee4e0, 0xc42d4ee540, 0xc429477878, 0x558012, 0x588a5743, 0x205750f8)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xed01c4e43, 0x0, 0x0, 0x5c59)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xc42d4ee4e0, 0xc42d4ee560, 0xa91b40, 0xc42d4ee580, 0xc42d4ee4e0, 0xc42d4ee580)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xc420246100, 0xc42d508030, 0xc429477a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d506030, 0xa934c0, 0xc42d506150, 0xc42d4ee4e0, 0xc42d4ee500, 0x48, 0xc42023bb18, 0x0, 0xc42023bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d506030, 0xc42d260900, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41fa28d46)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504000, 0xc420331380, 0x10265d8, 0xc42d5060f0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504000, 0xc42d5060f0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 1316 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc42048bb84)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc42048bb80)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x476db0d6341b0a2)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d7c82f0, 0x2, 0x2, 0xc4293df448, 0x0, 0xc42d6a1400)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d7864b0, 0x159dc604330, 0x159dc64d710, 0xc42d7c82f0, 0x2, 0x2, 0xc42d786400, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d7864b0, 0xc42d7863c0, 0x0, 0xc42d7c82c8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d7864b0, 0xa934c0, 0xc42d7863c0, 0x7fad6035c8e0, 0xc42d7864b0, 0x1038500, 0xc42d786480)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d786480, 0xa934c0, 0xc42d7863c0, 0xc42d73b260, 0xc42d73b280, 0x7fad6035c8e0, 0xc42d786480, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d786480, 0xa934c0, 0xc42d7863c0, 0x7fad6035c8e0, 0xc42d786480, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d786480, 0xa934c0, 0xc42d7863c0, 0xc42d73b260, 0xc42d73b2a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xffffffff, 0x0, 0x19, 0xc4293df7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xc42d73b260, 0xc42d73b2c0, 0xc4293df878, 0x558012, 0x588a5743, 0x2362a750)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xed01c4e43, 0x0, 0x0, 0x60c3)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xc42d73b260, 0xc42d73b2e0, 0xa91b40, 0xc42d73b300, 0xc42d73b260, 0xc42d73b300)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xc420246100, 0xc42d7c8280, 0xc4293dfa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d7862a0, 0xa934c0, 0xc42d7863c0, 0xc42d73b260, 0xc42d73b280, 0x48, 0xc425812b18, 0x0, 0xc425812b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d7862a0, 0xc42d260c60, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41fa28d46)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504600, 0xc420331380, 0x10265d8, 0xc42d786360, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504600, 0xc42d786360)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d504600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8738 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc443ff594c)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc443ff5940)
	/usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc443ff5940, 0xc443c341e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e

goroutine 1258 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204962c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204962c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x576fdc651f7c3092)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).forSeriesMatching(0xc42031cec0, 0xc42d351440, 0x2, 0x2, 0xc429109448, 0x0, 0xc42d386880)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:183 +0xea
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d3a4690, 0x159dc604330, 0x159dc64d710, 0xc42d351440, 0x2, 0x2, 0xc42d3a4600, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:291 +0x18a
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42d3a4690, 0xc42d3a45a0, 0x0, 0xc42d351418, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42d3a4690, 0xa934c0, 0xc42d3a45a0, 0x7fad6035c8e0, 0xc42d3a4690, 0x1038500, 0xc42d3a4660)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d3a4660, 0xa934c0, 0xc42d3a45a0, 0xc42d377ca0, 0xc42d377cc0, 0x7fad6035c8e0, 0xc42d3a4660, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a4660, 0xa934c0, 0xc42d3a45a0, 0x7fad6035c8e0, 0xc42d3a4660, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d3a4660, 0xa934c0, 0xc42d3a45a0, 0xc42d377ca0, 0xc42d377ce0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xffffffff, 0x0, 0x19, 0xc4291097e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xc42d377ca0, 0xc42d377d00, 0xc429109878, 0x558012, 0x588a5743, 0x1eba63a9)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xed01c4e43, 0x0, 0x0, 0x5311)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xc42d377ca0, 0xc42d377d20, 0xa91b40, 0xc42d377d40, 0xc42d377ca0, 0xc42d377d40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xc420246100, 0xc42d3513d0, 0xc429109a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d3a4480, 0xa934c0, 0xc42d3a45a0, 0xc42d377ca0, 0xc42d377cc0, 0x4e, 0xc420314b18, 0x0, 0xc420314b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d3a4480, 0xc42d1eaf00, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ece00, 0xc420331380, 0x10265d8, 0xc42d3a4540, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ece00, 0xc42d3a4540)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42939e040, 0xc4202460c0, 0x103ac20, 0xc426048300, 0xc42d1ece00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 5564 [select, 13 minutes]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc4271a6840)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 8552 [chan send, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc443d482d0, 0xc42e9d88a0, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 8865 [chan receive, 10 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc42443d220, 0xc42c0510e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc42443d220, 0xc42c0510e0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc42443d220, 0xc42c0510e0)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc44a36e360, 0xc42c0510e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc44a36e360, 0xc42c0510e0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc44a36e360, 0xc42c0510e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc44a36e340, 0xc42c050f00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc44a36e340, 0xc42c050f00)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc44a36e320, 0xc42c050f00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc44a36e320, 0xc42c050f00)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc43d0752b0, 0xc42c050f00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc43d0752b0, 0xc42c050e10)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc43d0752b0, 0xc42c050e10)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc4446eb300, 0x1038440, 0xc44628de40)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 9714 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4394e09f0, 0xc4394e0a20, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4394e09f0, 0xc43cf2a9c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4394e09f0, 0xc43c2a7ea0, 0x0, 0xc439ccf0c0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4394e09f0, 0xa74a80, 0xc43c2a7ea0, 0x7fad6035c8e0, 0xc4394e09f0, 0x1038500, 0xc4394e09c0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4394e09c0, 0xa74a80, 0xc43c2a7ea0, 0xc43cf2a8a0, 0xc43cf2a8c0, 0x7fad6035c8e0, 0xc4394e09c0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4394e09c0, 0xa74a80, 0xc43c2a7ea0, 0x7fad6035c8e0, 0xc4394e09c0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4394e09c0, 0xa74a80, 0xc43c2a7ea0, 0xc43cf2a8a0, 0xc43cf2a8e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xffffffff, 0x0, 0x19, 0xc449abf7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xc43cf2a8a0, 0xc43cf2a920, 0xc449abf878, 0x558012, 0x588a586d, 0x395d694c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xed01c4f6d, 0x0, 0x0, 0xa613d)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xc43cf2a8a0, 0xc43cf2a940, 0xa91b40, 0xc43cf2a960, 0xc43cf2a8a0, 0xc43cf2a960)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xc420246100, 0xc43c812a20, 0xc449abfa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43dff1680, 0xa74a80, 0xc43c2a7ea0, 0xc43cf2a8a0, 0xc43cf2a8c0, 0x37fa, 0xc420046b18, 0x0, 0xc420046b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43dff1680, 0xc42b6f39e0, 0xc420179d60, 0x0, 0x0, 0x7fad625dc000, 0xc41eac8542)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87d00, 0xc420331380, 0x10265c0, 0xc43dff16b0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87d00, 0xc43dff16b0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87d00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10277 [chan receive, 8 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc44060ec80, 0xc4200d5860)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc44060ec80, 0xc4200d5860)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc44060ec80, 0xc4200d5860)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc43e90fea0, 0xc4200d5860)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc43e90fea0, 0xc4200d5860)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc43e90fea0, 0xc4200d5860)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc43e90fe80, 0xc4200d5680)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc43e90fe80, 0xc4200d5680)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc43e90fe60, 0xc4200d5680)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc43e90fe60, 0xc4200d5680)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4201b40d0, 0xc4200d5680)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4201b40d0, 0xc4200d5590)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4201b40d0, 0xc4200d5590)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc430fd1580, 0x1038440, 0xc439995f80)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 8984 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4440a2690, 0xc4440a26c0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4440a2690, 0xc43a68d420, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4440a2690, 0xc43c59e980, 0x0, 0xc441187ea8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4440a2690, 0xa74a80, 0xc43c59e980, 0x7fad6035c8e0, 0xc4440a2690, 0x1038500, 0xc4440a2660)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4440a2660, 0xa74a80, 0xc43c59e980, 0xc43a68d300, 0xc43a68d320, 0x7fad6035c8e0, 0xc4440a2660, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4440a2660, 0xa74a80, 0xc43c59e980, 0x7fad6035c8e0, 0xc4440a2660, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4440a2660, 0xa74a80, 0xc43c59e980, 0xc43a68d300, 0xc43a68d360, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xffffffff, 0x0, 0x19, 0xc43d6cf7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xc43a68d300, 0xc43a68d380, 0xc43d6cf878, 0x558012, 0x588a5810, 0xd1afb8b)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xed01c4f10, 0x0, 0x0, 0xa55a3)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xc43a68d300, 0xc43a68d3a0, 0xa91b40, 0xc43a68d3c0, 0xc43a68d300, 0xc43a68d3c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xc420246100, 0xc4418b35d0, 0xc43d6cfa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43caf49f0, 0xa74a80, 0xc43c59e980, 0xc43a68d300, 0xc43a68d320, 0x7965, 0xc420e1db18, 0x0, 0xc420e1db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43caf49f0, 0xc4446d9da0, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0d00, 0xc420331380, 0x10265c0, 0xc43caf4b10, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0d00, 0xc43caf4b10)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a0d00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8800 [chan receive, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc443047040, 0xc430b3fa40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc443047040, 0xc430b3fa40)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc443047040, 0xc430b3fa40)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc431d568a0, 0xc430b3fa40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc431d568a0, 0xc430b3fa40)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc431d568a0, 0xc430b3fa40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc431d56880, 0xc430b3f860)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc431d56880, 0xc430b3f860)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc431d56860, 0xc430b3f860)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc431d56860, 0xc430b3f860)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4421cb5f0, 0xc430b3f860)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4421cb5f0, 0xc430b3f770)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4421cb5f0, 0xc430b3f770)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc448e5be80, 0x1038440, 0xc421344280)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 8861 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43bceed20, 0xc43bceed50, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43bceed20, 0xc42aa56dc0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43bceed20, 0xc442132a60, 0x0, 0xc4398152c8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43bceed20, 0xa74a80, 0xc442132a60, 0x7fad6035c8e0, 0xc43bceed20, 0x1038500, 0xc43bceecf0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43bceecf0, 0xa74a80, 0xc442132a60, 0xc42aa56cc0, 0xc42aa56ce0, 0x7fad6035c8e0, 0xc43bceecf0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43bceecf0, 0xa74a80, 0xc442132a60, 0x7fad6035c8e0, 0xc43bceecf0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43bceecf0, 0xa74a80, 0xc442132a60, 0xc42aa56cc0, 0xc42aa56d00, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xffffffff, 0x0, 0x19, 0xc44a87d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xc42aa56cc0, 0xc42aa56d20, 0xc44a87d878, 0x558012, 0x588a57f5, 0x3260f940)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xed01c4ef5, 0x0, 0x0, 0x12c304)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xc42aa56cc0, 0xc42aa56d40, 0xa91b40, 0xc42aa56d60, 0xc42aa56cc0, 0xc42aa56d60)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xc420246100, 0xc42b6f1ff0, 0xc44a87da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc448e5c270, 0xa74a80, 0xc442132a60, 0xc42aa56cc0, 0xc42aa56ce0, 0x90b4, 0xc420315b18, 0x0, 0xc420315b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc448e5c270, 0xc42a5f5800, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f1a9759)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87600, 0xc420331380, 0x10265c0, 0xc448e5c2a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87600, 0xc448e5c2a0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 12779 [semacquire]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42c690660, 0xc42c690690, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42c690660, 0xc43b978ca0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42c690660, 0xc44130b400, 0x0, 0xc42bcc9e48, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42c690660, 0xa74a80, 0xc44130b400, 0x7fad6035c8e0, 0xc42c690660, 0x1038500, 0xc42c690630)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c690630, 0xa74a80, 0xc44130b400, 0xc43b978b80, 0xc43b978ba0, 0x7fad6035c8e0, 0xc42c690630, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c690630, 0xa74a80, 0xc44130b400, 0x7fad6035c8e0, 0xc42c690630, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c690630, 0xa74a80, 0xc44130b400, 0xc43b978b80, 0xc43b978bc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0xffffffff, 0x0, 0x19, 0xc43132b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0xc43b978b80, 0xc43b978be0, 0xc43132b878, 0x558012, 0x588a5a4d, 0x3827c282)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0x41167e, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0xc43b978b80, 0xc43b978c00, 0xa91b40, 0xc43b978c40, 0xc43b978b80, 0xc43b978c40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0xc420246100, 0xc42d8d3990, 0xc43132ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d7d11d0, 0xa74a80, 0xc44130b400, 0xc43b978b80, 0xc43b978ba0, 0x7629, 0xc420e1cb18, 0x0, 0xc420e1cb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d7d11d0, 0xc440cd3b60, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f32cb4d)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1300, 0xc420331380, 0x10265c0, 0xc42d7d1200, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1300, 0xc42d7d1200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8405 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc42b868fc0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42b868fc0, 0x159d0153bd2, 0x159d04d14b2, 0xc428411f40, 0x1, 0x1, 0xc42b868f00, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:272 +0x7e
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42b868fc0, 0xc42b868f00, 0x0, 0xc42ba1ac00, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42b868fc0, 0xa934c0, 0xc42b868f00, 0x7fad6035c8e0, 0xc42b868fc0, 0x1038500, 0xc42b868f90)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42b868f90, 0xa934c0, 0xc42b868f00, 0xc435021120, 0xc435021140, 0x7fad6035c8e0, 0xc42b868f90, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b868f90, 0xa934c0, 0xc42b868f00, 0x7fad6035c8e0, 0xc42b868f90, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42b868f90, 0xa934c0, 0xc42b868f00, 0xc435021120, 0xc4350211a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xffffffff, 0x0, 0x19, 0xc4454c77e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xc435021120, 0xc4350211c0, 0xc4454c7878, 0x558012, 0x588a57a4, 0x367107c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xed01c4ea4, 0x0, 0x0, 0x17d1d5)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xc435021120, 0xc4350211e0, 0xa91b40, 0xc435021220, 0xc435021120, 0xc435021220)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xc420246100, 0xc42ba1abc0, 0xc4454c7a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42b868de0, 0xa934c0, 0xc42b868f00, 0xc435021120, 0xc435021140, 0x41, 0xc420e1cb18, 0x0, 0xc420e1cb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42b868de0, 0xc4275212c0, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc426516900, 0xc420331380, 0x10265d8, 0xc42b868ea0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc426516900, 0xc42b868ea0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc43e3ca700, 0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc426516900)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8649 [chan receive, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc42443d680, 0xc430b3e4b0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc42443d680, 0xc430b3e4b0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc42443d680, 0xc430b3e4b0)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc43333f920, 0xc430b3e4b0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc43333f920, 0xc430b3e4b0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc43333f920, 0xc430b3e4b0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc43333f8e0, 0xc430b3e2d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc43333f8e0, 0xc430b3e2d0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc43333f8c0, 0xc430b3e2d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc43333f8c0, 0xc430b3e2d0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc427d22f70, 0xc430b3e2d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc427d22f70, 0xc430b3e1e0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc427d22f70, 0xc430b3e1e0)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc43cb60300, 0x1038440, 0xc427adb300)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 8568 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43d29c360, 0xc43d29c390, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43d29c360, 0xc442b641e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43d29c360, 0xc43dcc7da0, 0x0, 0xc42e11d168, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43d29c360, 0xa74a80, 0xc43dcc7da0, 0x7fad6035c8e0, 0xc43d29c360, 0x1038500, 0xc43d29c330)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43d29c330, 0xa74a80, 0xc43dcc7da0, 0xc442b640e0, 0xc442b64100, 0x7fad6035c8e0, 0xc43d29c330, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d29c330, 0xa74a80, 0xc43dcc7da0, 0x7fad6035c8e0, 0xc43d29c330, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43d29c330, 0xa74a80, 0xc43dcc7da0, 0xc442b640e0, 0xc442b64120, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xffffffff, 0x0, 0x19, 0xc4389e57e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xc442b640e0, 0xc442b64140, 0xc4389e5878, 0x558012, 0x588a59b8, 0x382e051)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xed01c50b8, 0x0, 0x0, 0x11251d)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xc442b640e0, 0xc442b64160, 0xa91b40, 0xc442b64180, 0xc442b640e0, 0xc442b64180)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xc420246100, 0xc42d55b730, 0xc4389e5a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc440461890, 0xa74a80, 0xc43dcc7da0, 0xc442b640e0, 0xc442b64100, 0x4fe5, 0xc420e1db18, 0x0, 0xc420e1db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc440461890, 0xc43e024cc0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f700, 0xc420331380, 0x10265c0, 0xc4404618c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f700, 0xc4404618c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f700)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10660 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc432995dd0, 0xc432995ef0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc432995dd0, 0xc4448c7820, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc432995dd0, 0xc43efe8fc0, 0x0, 0xc435bf2050, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc432995dd0, 0xa74a80, 0xc43efe8fc0, 0x7fad6035c8e0, 0xc432995dd0, 0x1038500, 0xc432995cb0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc432995cb0, 0xa74a80, 0xc43efe8fc0, 0xc4448c7700, 0xc4448c7720, 0x7fad6035c8e0, 0xc432995cb0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc432995cb0, 0xa74a80, 0xc43efe8fc0, 0x7fad6035c8e0, 0xc432995cb0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc432995cb0, 0xa74a80, 0xc43efe8fc0, 0xc4448c7700, 0xc4448c7740, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xffffffff, 0x0, 0x19, 0xc424dab7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xc4448c7700, 0xc4448c7760, 0xc424dab878, 0x558012, 0x588a591e, 0x1000faff)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xed01c501e, 0x0, 0x0, 0x177897)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xc4448c7700, 0xc4448c7780, 0xa91b40, 0xc4448c77a0, 0xc4448c7700, 0xc4448c77a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xc420246100, 0xc435c37ab0, 0xc424daba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc432dde4e0, 0xa74a80, 0xc43efe8fc0, 0xc4448c7700, 0xc4448c7720, 0x46f6, 0xc42023bb18, 0x0, 0xc42023bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc432dde4e0, 0xc428920ae0, 0xc420179d60, 0x0, 0x0, 0x50, 0x4)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5d00, 0xc420331380, 0x10265c0, 0xc432dde570, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5d00, 0xc432dde570)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b5d00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8989 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43d26d620, 0xc43d26d650, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43d26d620, 0xc447e1ab20, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43d26d620, 0xc4481f7500, 0x0, 0xc43fc7dfc0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43d26d620, 0xa74a80, 0xc4481f7500, 0x7fad6035c8e0, 0xc43d26d620, 0x1038500, 0xc43d26d5f0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43d26d5f0, 0xa74a80, 0xc4481f7500, 0xc447e1a9e0, 0xc447e1aa20, 0x7fad6035c8e0, 0xc43d26d5f0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d26d5f0, 0xa74a80, 0xc4481f7500, 0x7fad6035c8e0, 0xc43d26d5f0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43d26d5f0, 0xa74a80, 0xc4481f7500, 0xc447e1a9e0, 0xc447e1aa40, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xffffffff, 0x0, 0x19, 0xc44a48d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xc447e1a9e0, 0xc447e1aa60, 0xc44a48d878, 0x558012, 0x588a5813, 0x36f6b00f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xed01c4f13, 0x0, 0x0, 0x126778)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xc447e1a9e0, 0xc447e1aa80, 0xa91b40, 0xc447e1aaa0, 0xc447e1a9e0, 0xc447e1aaa0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xc420246100, 0xc43fc7ce10, 0xc44a48da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc441afb290, 0xa74a80, 0xc4481f7500, 0xc447e1a9e0, 0xc447e1aa20, 0x2caa, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc441afb290, 0xc4431f7aa0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1c00, 0xc420331380, 0x10265c0, 0xc441afb2c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1c00, 0xc441afb2c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1c00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 11286 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a
github.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc448a02fc0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc4351a0d10, 0xc448a02fc0, 0x10343c0, 0xc4201e3040)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326

goroutine 8325 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42a56d7a0, 0xc42a56d7d0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42a56d7a0, 0xc437cba8a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42a56d7a0, 0xc437d3c020, 0x0, 0xc42a685c50, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42a56d7a0, 0xa74a80, 0xc437d3c020, 0x7fad6035c8e0, 0xc42a56d7a0, 0x1038500, 0xc42a56d770)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a56d770, 0xa74a80, 0xc437d3c020, 0xc437cba7a0, 0xc437cba7c0, 0x7fad6035c8e0, 0xc42a56d770, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a56d770, 0xa74a80, 0xc437d3c020, 0x7fad6035c8e0, 0xc42a56d770, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a56d770, 0xa74a80, 0xc437d3c020, 0xc437cba7a0, 0xc437cba7e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xffffffff, 0x0, 0x19, 0xc4359997e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xc437cba7a0, 0xc437cba800, 0xc435999878, 0x558012, 0x588a5798, 0xdb76d51)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xc435999900, 0x0, 0x0, 0x3f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xc437cba7a0, 0xc437cba820, 0xa91b40, 0xc437cba840, 0xc437cba7a0, 0xc437cba840)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xc420246100, 0xc4284342d0, 0xc435999a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4248f1d70, 0xa74a80, 0xc437d3c020, 0xc437cba7a0, 0xc437cba7c0, 0x9e4a, 0xc420e1ab18, 0x0, 0xc420e1ab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4248f1d70, 0xc448fe3680, 0xc420179d60, 0x0, 0x0, 0xc4202e17a8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3300, 0xc420331380, 0x10265c0, 0xc4248f1e00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3300, 0xc4248f1e00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4277c3300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8109 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc434146a50, 0xc434146a80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc434146a50, 0xc438f28420, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc434146a50, 0xc438f8d420, 0x0, 0xc4341417c8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc434146a50, 0xa74a80, 0xc438f8d420, 0x7fad6035c8e0, 0xc434146a50, 0x1038500, 0xc434146a20)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc434146a20, 0xa74a80, 0xc438f8d420, 0xc438f282e0, 0xc438f28300, 0x7fad6035c8e0, 0xc434146a20, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc434146a20, 0xa74a80, 0xc438f8d420, 0x7fad6035c8e0, 0xc434146a20, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc434146a20, 0xa74a80, 0xc438f8d420, 0xc438f282e0, 0xc438f28340, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xffffffff, 0x0, 0x19, 0xc4215df7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xc438f282e0, 0xc438f28360, 0xc4215df878, 0x558012, 0x588a577a, 0xac8b930)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xed01c4e7a, 0x0, 0x0, 0x72310)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xc438f282e0, 0xc438f28380, 0xa91b40, 0xc438f283a0, 0xc438f282e0, 0xc438f283a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xc420246100, 0xc42276fc80, 0xc4215dfa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44151e120, 0xa74a80, 0xc438f8d420, 0xc438f282e0, 0xc438f28300, 0x3e88, 0xc449a0ab18, 0x0, 0xc449a0ab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44151e120, 0xc423b1ec00, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa600, 0xc420331380, 0x10265c0, 0xc42516b200, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa600, 0xc42516b200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8323 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42c71a660, 0xc42c71a690, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42c71a660, 0xc437c84460, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42c71a660, 0xc437ce8f60, 0x0, 0xc42bc95ee0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42c71a660, 0xa74a80, 0xc437ce8f60, 0x7fad6035c8e0, 0xc42c71a660, 0x1038500, 0xc42c71a630)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c71a630, 0xa74a80, 0xc437ce8f60, 0xc437c84300, 0xc437c84320, 0x7fad6035c8e0, 0xc42c71a630, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c71a630, 0xa74a80, 0xc437ce8f60, 0x7fad6035c8e0, 0xc42c71a630, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c71a630, 0xa74a80, 0xc437ce8f60, 0xc437c84300, 0xc437c84340, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xffffffff, 0x0, 0x19, 0xc43b4ef7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xc437c84300, 0xc437c843a0, 0xc43b4ef878, 0x558012, 0x588a5798, 0xdbae98f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xed01c4e98, 0x0, 0x0, 0x2aa0b6)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xc437c84300, 0xc437c843c0, 0xa91b40, 0xc437c843e0, 0xc437c84300, 0xc437c843e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xc420246100, 0xc42a686a50, 0xc43b4efa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42f4079b0, 0xa74a80, 0xc437ce8f60, 0xc437c84300, 0xc437c84320, 0x859e, 0xc420049b18, 0x0, 0xc420049b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42f4079b0, 0xc4296fa6c0, 0xc420179d60, 0x0, 0x0, 0x646e696b040a100a, 0x61646174656d0812)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4277c2f00, 0xc420331380, 0x10265c0, 0xc42f4079e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4277c2f00, 0xc42f4079e0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4277c2f00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8985 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43e0c4e70, 0xc43e0c4ea0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43e0c4e70, 0xc43d8820e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43e0c4e70, 0xc4496c16a0, 0x0, 0xc4425b13c0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43e0c4e70, 0xa74a80, 0xc4496c16a0, 0x7fad6035c8e0, 0xc43e0c4e70, 0x1038500, 0xc43e0c4e40)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43e0c4e40, 0xa74a80, 0xc4496c16a0, 0xc443da9fe0, 0xc43d882000, 0x7fad6035c8e0, 0xc43e0c4e40, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c4e40, 0xa74a80, 0xc4496c16a0, 0x7fad6035c8e0, 0xc43e0c4e40, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43e0c4e40, 0xa74a80, 0xc4496c16a0, 0xc443da9fe0, 0xc43d882020, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xffffffff, 0x0, 0x19, 0xc44a66f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xc443da9fe0, 0xc43d882040, 0xc44a66f878, 0x558012, 0x588a5813, 0x328ff5c4)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xed01c4f13, 0x0, 0x0, 0x4665b)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xc443da9fe0, 0xc43d882060, 0xa91b40, 0xc43d882080, 0xc443da9fe0, 0xc43d882080)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xc420246100, 0xc4425b0920, 0xc44a66fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43e0c46c0, 0xa74a80, 0xc4496c16a0, 0xc443da9fe0, 0xc43d882000, 0x1ba7, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43e0c46c0, 0xc443c35800, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a1400, 0xc420331380, 0x10265c0, 0xc43e0c46f0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a1400, 0xc43e0c46f0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393a1400)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 5262 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc420497484)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc420497480)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x24560ccb27db9cc2)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc44240f1a0, 0xc422334c2d, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44240e3c0, 0xc44240f1a0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44240e3c0, 0xc42992e4e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44240e3c0, 0xc429a05260, 0x0, 0xc442410570, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44240e3c0, 0xa74a80, 0xc429a05260, 0x7fad6035c8e0, 0xc44240e3c0, 0x1038500, 0xc44240e390)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44240e390, 0xa74a80, 0xc429a05260, 0xc429961040, 0xc4299610a0, 0x7fad6035c8e0, 0xc44240e390, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44240e390, 0xa74a80, 0xc429a05260, 0x7fad6035c8e0, 0xc44240e390, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44240e390, 0xa74a80, 0xc429a05260, 0xc429961040, 0xc4299610c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xffffffff, 0x0, 0x19, 0xc42910d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xc429961040, 0xc4299610e0, 0xc42910d878, 0x558012, 0x588a575f, 0x3a21669b)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xed01c4e5f, 0x0, 0x0, 0x185c22)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xc429961040, 0xc429961120, 0xa91b40, 0xc429961140, 0xc429961040, 0xc429961140)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xc420246100, 0xc4435da830, 0xc42910da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43dff0c60, 0xa74a80, 0xc429a05260, 0xc429961040, 0xc4299610a0, 0xbace, 0xc449cdab18, 0x0, 0xc449cdab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43dff0c60, 0xc43bba5e60, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4eb00, 0xc420331380, 0x10265c0, 0xc43dff0c90, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4eb00, 0xc43dff0c90)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4eb00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8147 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4439c32f0, 0xc4439c3320, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4439c32f0, 0xc438b0e400, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4439c32f0, 0xc438b63b00, 0x0, 0xc448466cb0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4439c32f0, 0xa74a80, 0xc438b63b00, 0x7fad6035c8e0, 0xc4439c32f0, 0x1038500, 0xc4439c32c0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4439c32c0, 0xa74a80, 0xc438b63b00, 0xc438b0e2a0, 0xc438b0e2c0, 0x7fad6035c8e0, 0xc4439c32c0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4439c32c0, 0xa74a80, 0xc438b63b00, 0x7fad6035c8e0, 0xc4439c32c0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4439c32c0, 0xa74a80, 0xc438b63b00, 0xc438b0e2a0, 0xc438b0e2e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xffffffff, 0x0, 0x19, 0xc43d1517e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xc438b0e2a0, 0xc438b0e300, 0xc43d151878, 0x558012, 0x588a577d, 0x3873ef2f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xed01c4e7d, 0x0, 0x0, 0xe3b6e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xc438b0e2a0, 0xc438b0e320, 0xa91b40, 0xc438b0e360, 0xc438b0e2a0, 0xc438b0e360)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xc420246100, 0xc44757c9c0, 0xc43d151a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442346660, 0xa74a80, 0xc438b63b00, 0xc438b0e2a0, 0xc438b0e2c0, 0xa28a, 0xc420319b18, 0x0, 0xc420319b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442346660, 0xc43d52d740, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abf00, 0xc420331380, 0x10265c0, 0xc442346690, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abf00, 0xc442346690)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267abf00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10458 [chan send, 6 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc42c051590, 0xc438e702a0, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 11262 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d097860, 0xc42d097890, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d097860, 0xc443553c80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d097860, 0xc446b26080, 0x0, 0xc4333feaf8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d097860, 0xa74a80, 0xc446b26080, 0x7fad6035c8e0, 0xc42d097860, 0x1038500, 0xc42d097830)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d097830, 0xa74a80, 0xc446b26080, 0xc443553b80, 0xc443553ba0, 0x7fad6035c8e0, 0xc42d097830, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d097830, 0xa74a80, 0xc446b26080, 0x7fad6035c8e0, 0xc42d097830, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d097830, 0xa74a80, 0xc446b26080, 0xc443553b80, 0xc443553bc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0xffffffff, 0x0, 0x19, 0xc4491af7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0xc443553b80, 0xc443553be0, 0xc4491af878, 0x558012, 0x588a5978, 0x11c865da)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0x41167e, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0xc443553b80, 0xc443553c00, 0xa91b40, 0xc443553c20, 0xc443553b80, 0xc443553c20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0xc420246100, 0xc4335ea6b0, 0xc4491afa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42b3ad230, 0xa74a80, 0xc446b26080, 0xc443553b80, 0xc443553ba0, 0x75ed, 0xc440d64b18, 0x0, 0xc440d64b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42b3ad230, 0xc44415d860, 0xc420179d60, 0x0, 0x0, 0x10, 0xc439ad6e58)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fb300, 0xc420331380, 0x10265c0, 0xc42b3ad2f0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fb300, 0xc42b3ad2f0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fb300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8113 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc440d230e0, 0xc440d23110, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc440d230e0, 0xc438dafe60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc440d230e0, 0xc438de3a60, 0x0, 0xc440d24190, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc440d230e0, 0xa74a80, 0xc438de3a60, 0x7fad6035c8e0, 0xc440d230e0, 0x1038500, 0xc440d230b0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc440d230b0, 0xa74a80, 0xc438de3a60, 0xc438dafd20, 0xc438dafd40, 0x7fad6035c8e0, 0xc440d230b0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440d230b0, 0xa74a80, 0xc438de3a60, 0x7fad6035c8e0, 0xc440d230b0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc440d230b0, 0xa74a80, 0xc438de3a60, 0xc438dafd20, 0xc438dafd60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xffffffff, 0x0, 0x19, 0xc42d70f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xc438dafd20, 0xc438dafda0, 0xc42d70f878, 0x558012, 0x588a577a, 0xcf101a6)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xed01c4e7a, 0x0, 0x0, 0x692eb)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xc438dafd20, 0xc438dafdc0, 0xa91b40, 0xc438dafde0, 0xc438dafd20, 0xc438dafde0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xc420246100, 0xc446b14800, 0xc42d70fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc446b13a70, 0xa74a80, 0xc438de3a60, 0xc438dafd20, 0xc438dafd40, 0x3746, 0xc420044b18, 0x0, 0xc420044b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc446b13a70, 0xc446b0c120, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4267ab300, 0xc420331380, 0x10265c0, 0xc446b13b30, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4267ab300, 0xc446b13b30)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4267ab300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8073 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc423f8c600, 0xc423f8c630, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc423f8c600, 0xc433e06380, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc423f8c600, 0xc433f2bbe0, 0x0, 0xc423f8e140, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc423f8c600, 0xa74a80, 0xc433f2bbe0, 0x7fad6035c8e0, 0xc423f8c600, 0x1038500, 0xc423f8c5d0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc423f8c5d0, 0xa74a80, 0xc433f2bbe0, 0xc433e06180, 0xc433e061e0, 0x7fad6035c8e0, 0xc423f8c5d0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423f8c5d0, 0xa74a80, 0xc433f2bbe0, 0x7fad6035c8e0, 0xc423f8c5d0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc423f8c5d0, 0xa74a80, 0xc433f2bbe0, 0xc433e06180, 0xc433e06200, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xffffffff, 0x0, 0x19, 0xc448f057e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xc433e06180, 0xc433e06220, 0xc448f05878, 0x558012, 0x588a57b6, 0xd019425)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xed01c4eb6, 0x0, 0x0, 0xc1691)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xc433e06180, 0xc433e062a0, 0xa91b40, 0xc433e062c0, 0xc433e06180, 0xc433e062c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xc420246100, 0xc424ba9bc0, 0xc448f05a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42378aae0, 0xa74a80, 0xc433f2bbe0, 0xc433e06180, 0xc433e061e0, 0x66ce, 0xc420319b18, 0x0, 0xc420319b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42378aae0, 0xc42779ec60, 0xc420179d60, 0x0, 0x0, 0x4d1279726f746973, 0x6f642e7865646e69)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4228bc600, 0xc420331380, 0x10265c0, 0xc42378ab10, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4228bc600, 0xc42378ab10)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4228bc600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8988 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441afb560, 0xc441afb590, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441afb560, 0xc44693fc00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441afb560, 0xc44693f700, 0x0, 0xc448040940, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441afb560, 0xa74a80, 0xc44693f700, 0x7fad6035c8e0, 0xc441afb560, 0x1038500, 0xc441afb530)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc441afb530, 0xa74a80, 0xc44693f700, 0xc44693fae0, 0xc44693fb00, 0x7fad6035c8e0, 0xc441afb530, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb530, 0xa74a80, 0xc44693f700, 0x7fad6035c8e0, 0xc441afb530, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc441afb530, 0xa74a80, 0xc44693f700, 0xc44693fae0, 0xc44693fb20, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xffffffff, 0x0, 0x19, 0xc44a48f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xc44693fae0, 0xc44693fb40, 0xc44a48f878, 0x558012, 0x588a5813, 0x36e5e965)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xed01c4f13, 0x0, 0x0, 0x11cf1)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xc44693fae0, 0xc44693fb60, 0xa91b40, 0xc44693fba0, 0xc44693fae0, 0xc44693fba0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xc420246100, 0xc448040050, 0xc44a48fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc441afb0b0, 0xa74a80, 0xc44693f700, 0xc44693fae0, 0xc44693fb00, 0x272, 0xc420e1db18, 0x0, 0xc420e1db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc441afb0b0, 0xc443c35e00, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1b00, 0xc420331380, 0x10265c0, 0xc441afb320, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1b00, 0xc441afb320)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1b00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9883 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437c77920, 0xc437c779b0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437c77920, 0xc44104b6c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437c77920, 0xc445c70b00, 0x0, 0xc438b972b0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437c77920, 0xa74a80, 0xc445c70b00, 0x7fad6035c8e0, 0xc437c77920, 0x1038500, 0xc437c77890)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc437c77890, 0xa74a80, 0xc445c70b00, 0xc44104b5c0, 0xc44104b5e0, 0x7fad6035c8e0, 0xc437c77890, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c77890, 0xa74a80, 0xc445c70b00, 0x7fad6035c8e0, 0xc437c77890, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc437c77890, 0xa74a80, 0xc445c70b00, 0xc44104b5c0, 0xc44104b600, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xffffffff, 0x0, 0x19, 0xc44145b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xc44104b5c0, 0xc44104b620, 0xc44145b878, 0x558012, 0x588a588b, 0x32d2020d)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xed01c4f8b, 0x0, 0x0, 0x8457f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xc44104b5c0, 0xc44104b640, 0xa91b40, 0xc44104b660, 0xc44104b5c0, 0xc44104b660)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xc420246100, 0xc438c390c0, 0xc44145ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc437cc1800, 0xa74a80, 0xc445c70b00, 0xc44104b5c0, 0xc44104b5e0, 0x2edd, 0xc42023bb18, 0x0, 0xc42023bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc437cc1800, 0xc4411971a0, 0xc420179d60, 0x0, 0x0, 0xc439252d40, 0x646e696b)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b5c00, 0xc420331380, 0x10265c0, 0xc437cc1830, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b5c00, 0xc437cc1830)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4393b5c00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8518 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42210b020, 0xc42210b050, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42210b020, 0xc433b7ab00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42210b020, 0xc433bba280, 0x0, 0xc42b6f0760, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42210b020, 0xa74a80, 0xc433bba280, 0x7fad6035c8e0, 0xc42210b020, 0x1038500, 0xc42210aff0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42210aff0, 0xa74a80, 0xc433bba280, 0xc433b7a9a0, 0xc433b7a9c0, 0x7fad6035c8e0, 0xc42210aff0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210aff0, 0xa74a80, 0xc433bba280, 0x7fad6035c8e0, 0xc42210aff0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42210aff0, 0xa74a80, 0xc433bba280, 0xc433b7a9a0, 0xc433b7aa20, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xffffffff, 0x0, 0x19, 0xc4425a77e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xc433b7a9a0, 0xc433b7aa40, 0xc4425a7878, 0x558012, 0x588a57b9, 0x3b697901)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xed01c4eb9, 0x0, 0x0, 0x2ce5d)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xc433b7a9a0, 0xc433b7aa60, 0xa91b40, 0xc433b7aa80, 0xc433b7a9a0, 0xc433b7aa80)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xc420246100, 0xc4223f7d60, 0xc4425a7a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42210a8d0, 0xa74a80, 0xc433bba280, 0xc433b7a9a0, 0xc433b7a9c0, 0x1aba, 0xc420318b18, 0x0, 0xc420318b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42210a8d0, 0xc428757f20, 0xc420179d60, 0x0, 0x0, 0x4d1279726f746973, 0x6f642e7865646e69)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f100, 0xc420331380, 0x10265c0, 0xc42210a990, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f100, 0xc42210a990)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f100)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8531 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4269fc3c0, 0xc4269fc3f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4269fc3c0, 0xc434061f00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4269fc3c0, 0xc434200bc0, 0x0, 0xc426a916c0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4269fc3c0, 0xa74a80, 0xc434200bc0, 0x7fad6035c8e0, 0xc4269fc3c0, 0x1038500, 0xc4269fc390)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4269fc390, 0xa74a80, 0xc434200bc0, 0xc434061da0, 0xc434061de0, 0x7fad6035c8e0, 0xc4269fc390, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4269fc390, 0xa74a80, 0xc434200bc0, 0x7fad6035c8e0, 0xc4269fc390, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4269fc390, 0xa74a80, 0xc434200bc0, 0xc434061da0, 0xc434061e00, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xffffffff, 0x0, 0x19, 0xc43fccd7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xc434061da0, 0xc434061e20, 0xc43fccd878, 0x558012, 0x588a57b6, 0xbef1596)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xed01c4eb6, 0x0, 0x0, 0x121c53)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xc434061da0, 0xc434061e60, 0xa91b40, 0xc434061e80, 0xc434061da0, 0xc434061e80)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xc420246100, 0xc4284b1a60, 0xc43fccda70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc428928c60, 0xa74a80, 0xc434200bc0, 0xc434061da0, 0xc434061de0, 0xaf25, 0xc420e18b18, 0x0, 0xc420e18b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc428928c60, 0xc428756720, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5d1194)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422e08500, 0xc420331380, 0x10265c0, 0xc428928c90, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422e08500, 0xc428928c90)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422e08500)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10239 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43627f2c0, 0xc43627f350, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43627f2c0, 0xc43b608220, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43627f2c0, 0xc43b257320, 0x0, 0xc43772da00, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43627f2c0, 0xa74a80, 0xc43b257320, 0x7fad6035c8e0, 0xc43627f2c0, 0x1038500, 0xc43627f230)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43627f230, 0xa74a80, 0xc43b257320, 0xc43b608120, 0xc43b608140, 0x7fad6035c8e0, 0xc43627f230, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43627f230, 0xa74a80, 0xc43b257320, 0x7fad6035c8e0, 0xc43627f230, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43627f230, 0xa74a80, 0xc43b257320, 0xc43b608120, 0xc43b608160, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xffffffff, 0x0, 0x19, 0xc43b0937e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xc43b608120, 0xc43b608180, 0xc43b093878, 0x558012, 0x588a58c4, 0x10f2f5bc)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xed01c4fc4, 0x0, 0x0, 0x13506b)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xc43b608120, 0xc43b6081a0, 0xa91b40, 0xc43b6081c0, 0xc43b608120, 0xc43b6081c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xc420246100, 0xc4377ba030, 0xc43b093a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4362ce720, 0xa74a80, 0xc43b257320, 0xc43b608120, 0xc43b608140, 0x62d8, 0xc420239b18, 0x0, 0xc420239b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4362ce720, 0xc433d43020, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438723b00, 0xc420331380, 0x10265c0, 0xc4362ced50, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438723b00, 0xc4362ced50)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438723b00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 7441 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44175c660, 0xc44175c690, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44175c660, 0xc447b4e520, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44175c660, 0xc44177a020, 0x0, 0xc432bf7c80, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44175c660, 0xa74a80, 0xc44177a020, 0x7fad6035c8e0, 0xc44175c660, 0x1038500, 0xc44175c630)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44175c630, 0xa74a80, 0xc44177a020, 0xc447b4e3e0, 0xc447b4e400, 0x7fad6035c8e0, 0xc44175c630, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44175c630, 0xa74a80, 0xc44177a020, 0x7fad6035c8e0, 0xc44175c630, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44175c630, 0xa74a80, 0xc44177a020, 0xc447b4e3e0, 0xc447b4e420, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xffffffff, 0x0, 0x19, 0xc43808b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xc447b4e3e0, 0xc447b4e440, 0xc43808b878, 0x558012, 0x588a59d2, 0x1224f8ee)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xed01c50d2, 0x0, 0x0, 0x1827c8)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xc447b4e3e0, 0xc447b4e460, 0xa91b40, 0xc447b4e480, 0xc447b4e3e0, 0xc447b4e480)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xc420246100, 0xc4351c4880, 0xc43808ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4424521b0, 0xa74a80, 0xc44177a020, 0xc447b4e3e0, 0xc447b4e400, 0x7c56, 0xc42023cb18, 0x0, 0xc42023cb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4424521b0, 0xc44103c180, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ed5c013)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6000, 0xc420331380, 0x10265c0, 0xc4424524e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6000, 0xc4424524e0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 5563 [IO wait, 4 minutes]:
net.runtime_pollWait(0x7fad6039cca0, 0x72, 0xc)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc43d5271e0, 0x72, 0xc4203178d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc43d5271e0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc43d527180, 0xc43e0d6000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc43a4a07d8, 0xc43e0d6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc43e447a40)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc43e447a40, 0xc4271a67b8, 0x9, 0x9, 0x43fe80, 0xc431af8170, 0x0)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc43e447a40, 0xc4271a67b8, 0x9, 0x9, 0x9, 0xc420317ae8, 0x451a90, 0xc43ba02dc0)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc43e447a40, 0xc4271a67b8, 0x9, 0x9, 0xc431af8170, 0x0, 0x1)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc4271a67b8, 0x9, 0x9, 0x102f380, 0xc43e447a40, 0x0, 0xc400000000, 0xc420317c80, 0x7f3105)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4271a6780, 0xf, 0xf, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc43e0c5410, 0xc43b9713e0, 0xc43b9713e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4271a6840, 0xc43e0c54a0, 0xb475b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc4271a6840)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc43a4a07d8, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc43a4a07d8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 12131 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a
github.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc443068600)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc42e0006f0, 0xc443068600, 0x10343c0, 0xc4201e3040)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326

goroutine 8986 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441afa270, 0xc441afa2a0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441afa270, 0xc44693ec00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441afa270, 0xc43d882460, 0x0, 0xc449c55ea0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441afa270, 0xa74a80, 0xc43d882460, 0x7fad6035c8e0, 0xc441afa270, 0x1038500, 0xc441afa240)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc441afa240, 0xa74a80, 0xc43d882460, 0xc44693eae0, 0xc44693eb00, 0x7fad6035c8e0, 0xc441afa240, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441afa240, 0xa74a80, 0xc43d882460, 0x7fad6035c8e0, 0xc441afa240, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc441afa240, 0xa74a80, 0xc43d882460, 0xc44693eae0, 0xc44693eb20, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xffffffff, 0x0, 0x19, 0xc44a6717e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xc44693eae0, 0xc44693eb40, 0xc44a671878, 0x558012, 0x588a5813, 0x331cd64c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xed01c4f13, 0x0, 0x0, 0x67995)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xc44693eae0, 0xc44693eb60, 0xa91b40, 0xc44693eb80, 0xc44693eae0, 0xc44693eb80)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xc420246100, 0xc4425b1470, 0xc44a671a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43e0c5530, 0xa74a80, 0xc43d882460, 0xc44693eae0, 0xc44693eb00, 0x29ae, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43e0c5530, 0xc443c35b60, 0xc420179d60, 0x0, 0x0, 0xc4200e5800, 0x450280)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1700, 0xc420331380, 0x10265c0, 0xc43e0c5560, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1700, 0xc43e0c5560)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a1700)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 5346 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204895c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204895c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x8ec308e871713ce)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc444ae6f30, 0xc420162418, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc444ae65d0, 0xc444ae6f30, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc444ae65d0, 0xc42b4b1660, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc444ae65d0, 0xc42b53f240, 0x0, 0xc444aea400, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc444ae65d0, 0xa74a80, 0xc42b53f240, 0x7fad6035c8e0, 0xc444ae65d0, 0x1038500, 0xc444ae65a0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc444ae65a0, 0xa74a80, 0xc42b53f240, 0xc42b4b0b60, 0xc42b4b0ba0, 0x7fad6035c8e0, 0xc444ae65a0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444ae65a0, 0xa74a80, 0xc42b53f240, 0x7fad6035c8e0, 0xc444ae65a0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc444ae65a0, 0xa74a80, 0xc42b53f240, 0xc42b4b0b60, 0xc42b4b0bc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xffffffff, 0x0, 0x19, 0xc4215bd7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xc42b4b0b60, 0xc42b4b0be0, 0xc4215bd878, 0x558012, 0x588a575f, 0x36ab2c69)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xed01c4e5f, 0x0, 0x0, 0xf2bf3)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xc42b4b0b60, 0xc42b4b0c20, 0xa91b40, 0xc42b4b0c40, 0xc42b4b0b60, 0xc42b4b0c40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xc420246100, 0xc444f1f710, 0xc4215bda70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc444dba7e0, 0xa74a80, 0xc42b53f240, 0xc42b4b0b60, 0xc42b4b0ba0, 0x8a5c, 0xc420313b18, 0x0, 0xc420313b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc444dba7e0, 0xc448c5cc60, 0xc420179d60, 0x0, 0x0, 0xc42ad71fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbef00, 0xc420331380, 0x10265c0, 0xc444dba810, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbef00, 0xc444dba810)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbef00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8982 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43b0b6780, 0xc43b0b67b0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43b0b6780, 0xc43f6e7f40, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43b0b6780, 0xc43aef1fc0, 0x0, 0xc442d71580, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43b0b6780, 0xa74a80, 0xc43aef1fc0, 0x7fad6035c8e0, 0xc43b0b6780, 0x1038500, 0xc43b0b6750)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43b0b6750, 0xa74a80, 0xc43aef1fc0, 0xc43f6e7e40, 0xc43f6e7e60, 0x7fad6035c8e0, 0xc43b0b6750, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b0b6750, 0xa74a80, 0xc43aef1fc0, 0x7fad6035c8e0, 0xc43b0b6750, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43b0b6750, 0xa74a80, 0xc43aef1fc0, 0xc43f6e7e40, 0xc43f6e7e80, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xffffffff, 0x0, 0x19, 0xc43d6d37e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xc43f6e7e40, 0xc43f6e7ea0, 0xc43d6d3878, 0x558012, 0x588a5810, 0xd2c314c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xed01c4f10, 0x0, 0x0, 0x2db06e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xc43f6e7e40, 0xc43f6e7ec0, 0xa91b40, 0xc43f6e7ee0, 0xc43f6e7e40, 0xc43f6e7ee0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xc420246100, 0xc441e55320, 0xc43d6d3a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43caf41b0, 0xa74a80, 0xc43aef1fc0, 0xc43f6e7e40, 0xc43f6e7e60, 0x75b3, 0xc420e18b18, 0x0, 0xc420e18b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43caf41b0, 0xc4431f7320, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f3c36c4)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0800, 0xc420331380, 0x10265c0, 0xc43caf4450, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0800, 0xc43caf4450)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393a0800)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8569 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43fab54d0, 0xc43fab5500, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43fab54d0, 0xc43dad9d60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43fab54d0, 0xc447b4f340, 0x0, 0xc431af9238, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43fab54d0, 0xa74a80, 0xc447b4f340, 0x7fad6035c8e0, 0xc43fab54d0, 0x1038500, 0xc43fab54a0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43fab54a0, 0xa74a80, 0xc447b4f340, 0xc43dad9c60, 0xc43dad9c80, 0x7fad6035c8e0, 0xc43fab54a0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43fab54a0, 0xa74a80, 0xc447b4f340, 0x7fad6035c8e0, 0xc43fab54a0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43fab54a0, 0xa74a80, 0xc447b4f340, 0xc43dad9c60, 0xc43dad9ca0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xffffffff, 0x0, 0x19, 0xc4383637e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xc43dad9c60, 0xc43dad9cc0, 0xc438363878, 0x558012, 0x588a59d2, 0x123bfb30)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xed01c50d2, 0x0, 0x0, 0x29b015)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xc43dad9c60, 0xc43dad9ce0, 0xa91b40, 0xc43dad9d00, 0xc43dad9c60, 0xc43dad9d00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xc420246100, 0xc4329c4260, 0xc438363a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4432da030, 0xa74a80, 0xc447b4f340, 0xc43dad9c60, 0xc43dad9c80, 0x799d, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4432da030, 0xc44103c1e0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f228f32)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439775500, 0xc420331380, 0x10265c0, 0xc4432da060, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439775500, 0xc4432da060)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439775500)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 5378 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc420489c04)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc420489c00)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x46e19a0f49d894d)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc4416303f0, 0xc422334c2d, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441630330, 0xc4416303f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441630330, 0xc42935f6e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441630330, 0xc429429c60, 0x0, 0xc4418654e8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441630330, 0xa74a80, 0xc429429c60, 0x7fad6035c8e0, 0xc441630330, 0x1038500, 0xc441630300)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc441630300, 0xa74a80, 0xc429429c60, 0xc42935f0e0, 0xc42935f140, 0x7fad6035c8e0, 0xc441630300, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441630300, 0xa74a80, 0xc429429c60, 0x7fad6035c8e0, 0xc441630300, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc441630300, 0xa74a80, 0xc429429c60, 0xc42935f0e0, 0xc42935f180, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xffffffff, 0x0, 0x19, 0xc43ee3f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xc42935f0e0, 0xc42935f2c0, 0xc43ee3f878, 0x558012, 0x588a5760, 0x215ec54)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xed01c4e60, 0x0, 0x0, 0x100641)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xc42935f0e0, 0xc42935f300, 0xa91b40, 0xc42935f380, 0xc42935f0e0, 0xc42935f380)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xc420246100, 0xc4418b3970, 0xc43ee3fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4418540f0, 0xa74a80, 0xc429429c60, 0xc42935f0e0, 0xc42935f140, 0xabbf, 0xc446d5eb18, 0x0, 0xc446d5eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4418540f0, 0xc440870fc0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f600, 0xc420331380, 0x10265c0, 0xc441854120, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f600, 0xc441854120)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4f600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10272 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc435b86420, 0xc435b871d0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc435b86420, 0xc4428c9da0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc435b86420, 0xc445d3f340, 0x0, 0xc43732bcb0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc435b86420, 0xa74a80, 0xc445d3f340, 0x7fad6035c8e0, 0xc435b86420, 0x1038500, 0xc435b863f0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc435b863f0, 0xa74a80, 0xc445d3f340, 0xc4428c9ca0, 0xc4428c9cc0, 0x7fad6035c8e0, 0xc435b863f0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b863f0, 0xa74a80, 0xc445d3f340, 0x7fad6035c8e0, 0xc435b863f0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc435b863f0, 0xa74a80, 0xc445d3f340, 0xc4428c9ca0, 0xc4428c9ce0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xffffffff, 0x0, 0x19, 0xc440b4b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xc4428c9ca0, 0xc4428c9d00, 0xc440b4b878, 0x558012, 0x588a58c7, 0x3389e516)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xed01c4fc7, 0x0, 0x0, 0x275370)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xc4428c9ca0, 0xc4428c9d20, 0xa91b40, 0xc4428c9d40, 0xc4428c9ca0, 0xc4428c9d40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xc420246100, 0xc43748c240, 0xc440b4ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc435bdfda0, 0xa74a80, 0xc445d3f340, 0xc4428c9ca0, 0xc4428c9cc0, 0xabd9, 0xc420319b18, 0x0, 0xc420319b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc435bdfda0, 0xc433c3ec00, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ee82460)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc000, 0xc420331380, 0x10265c0, 0xc435bdfdd0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc000, 0xc435bdfdd0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8326 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4223bc990, 0xc4223bc9c0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4223bc990, 0xc437b94840, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4223bc990, 0xc437bd6340, 0x0, 0xc4223bb490, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4223bc990, 0xa74a80, 0xc437bd6340, 0x7fad6035c8e0, 0xc4223bc990, 0x1038500, 0xc4223bc960)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4223bc960, 0xa74a80, 0xc437bd6340, 0xc437b94700, 0xc437b94720, 0x7fad6035c8e0, 0xc4223bc960, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4223bc960, 0xa74a80, 0xc437bd6340, 0x7fad6035c8e0, 0xc4223bc960, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4223bc960, 0xa74a80, 0xc437bd6340, 0xc437b94700, 0xc437b94740, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xffffffff, 0x0, 0x19, 0xc4454c57e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xc437b94700, 0xc437b94780, 0xc4454c5878, 0x558012, 0x588a579b, 0x32a304c1)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xed01c4e9b, 0x0, 0x0, 0x1871af)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xc437b94700, 0xc437b947a0, 0xa91b40, 0xc437b947c0, 0xc437b94700, 0xc437b947c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xc420246100, 0xc42df97ad0, 0xc4454c5a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42deb6510, 0xa74a80, 0xc437bd6340, 0xc437b94700, 0xc437b94720, 0x8bd0, 0xc420e1db18, 0x0, 0xc420e1db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42deb6510, 0xc4296fb740, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f40378e)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516100, 0xc420331380, 0x10265c0, 0xc42deb6540, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516100, 0xc42deb6540)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc426516100)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8151 [chan receive, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc42d97bf40, 0xc42c0502d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc42d97bf40, 0xc42c0502d0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc42d97bf40, 0xc42c0502d0)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc43a5a1dc0, 0xc42c0502d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc43a5a1dc0, 0xc42c0502d0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc43a5a1dc0, 0xc42c0502d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc43a5a1da0, 0xc42c0500f0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc43a5a1da0, 0xc42c0500f0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc43a5a1d80, 0xc42c0500f0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc43a5a1d80, 0xc42c0500f0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4292de340, 0xc42c0500f0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4292de340, 0xc42c050000)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4292de340, 0xc42c050000)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc43e55b580, 0x1038440, 0xc42b140bc0)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 8930 [chan send, 10 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc42c0510e0, 0xc445cf80c0, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 8391 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a
github.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc42c212900)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc42c133760, 0xc42c212900, 0x10343c0, 0xc4201e3040)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326

goroutine 8242 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc444b24d4c)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc444b24d40)
	/usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc444b24d40, 0xc446b0daa0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e

goroutine 8747 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a
github.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc443c341e0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc443ff5940, 0xc443c341e0, 0x10343c0, 0xc4201e3040)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326

goroutine 10276 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43cbb6690, 0xc43cbb66c0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43cbb6690, 0xc43e90f300, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43cbb6690, 0xc443e91ce0, 0x0, 0xc436eb9ff8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43cbb6690, 0xa74a80, 0xc443e91ce0, 0x7fad6035c8e0, 0xc43cbb6690, 0x1038500, 0xc43cbb6660)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43cbb6660, 0xa74a80, 0xc443e91ce0, 0xc43e90f1e0, 0xc43e90f200, 0x7fad6035c8e0, 0xc43cbb6660, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43cbb6660, 0xa74a80, 0xc443e91ce0, 0x7fad6035c8e0, 0xc43cbb6660, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43cbb6660, 0xa74a80, 0xc443e91ce0, 0xc43e90f1e0, 0xc43e90f220, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xffffffff, 0x0, 0x19, 0xc442c1f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xc43e90f1e0, 0xc43e90f240, 0xc442c1f878, 0x558012, 0x588a58c8, 0x527f27a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xed01c4fc8, 0x0, 0x0, 0xb19bf)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xc43e90f1e0, 0xc43e90f260, 0xa91b40, 0xc43e90f280, 0xc43e90f1e0, 0xc43e90f280)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xc420246100, 0xc437023ae0, 0xc442c1fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43564f770, 0xa74a80, 0xc443e91ce0, 0xc43e90f1e0, 0xc43e90f200, 0x8e85, 0xc420318b18, 0x0, 0xc420318b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43564f770, 0xc42fba9aa0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdcb00, 0xc420331380, 0x10265c0, 0xc43564f830, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdcb00, 0xc43564f830)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdcb00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8549 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4241a7530, 0xc4241a7560, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4241a7530, 0xc433c72aa0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4241a7530, 0xc4415dd160, 0x0, 0xc42443bae0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4241a7530, 0xa74a80, 0xc4415dd160, 0x7fad6035c8e0, 0xc4241a7530, 0x1038500, 0xc4241a7500)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4241a7500, 0xa74a80, 0xc4415dd160, 0xc433c72980, 0xc433c729a0, 0x7fad6035c8e0, 0xc4241a7500, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a7500, 0xa74a80, 0xc4415dd160, 0x7fad6035c8e0, 0xc4241a7500, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4241a7500, 0xa74a80, 0xc4415dd160, 0xc433c72980, 0xc433c729c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xffffffff, 0x0, 0x19, 0xc448c2d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xc433c72980, 0xc433c729e0, 0xc448c2d878, 0x558012, 0x588a57b9, 0x38c61ff2)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xed01c4eb9, 0x0, 0x0, 0x2aa54)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xc433c72980, 0xc433c72a00, 0xa91b40, 0xc433c72a20, 0xc433c72980, 0xc433c72a20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xc420246100, 0xc42443b170, 0xc448c2da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4241a6e40, 0xa74a80, 0xc4415dd160, 0xc433c72980, 0xc433c729a0, 0x17d5, 0xc420047b18, 0x0, 0xc420047b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4241a6e40, 0xc42693d740, 0xc420179d60, 0x0, 0x0, 0xc420fa6fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc425cf3700, 0xc420331380, 0x10265c0, 0xc4241a6f00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc425cf3700, 0xc4241a6f00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc425cf3700)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 5575 [select, 13 minutes]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc42000d740)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 5350 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc420489c04)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc420489c00)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x9e919d062dc9c8bf)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc43e303410, 0xc420162418, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43e3030b0, 0xc43e303410, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43e3030b0, 0xc428a42ba0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43e3030b0, 0xc428ccd620, 0x0, 0xc43e30b170, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43e3030b0, 0xa74a80, 0xc428ccd620, 0x7fad6035c8e0, 0xc43e3030b0, 0x1038500, 0xc43e303080)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43e303080, 0xa74a80, 0xc428ccd620, 0xc428a422a0, 0xc428a42360, 0x7fad6035c8e0, 0xc43e303080, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e303080, 0xa74a80, 0xc428ccd620, 0x7fad6035c8e0, 0xc43e303080, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43e303080, 0xa74a80, 0xc428ccd620, 0xc428a422a0, 0xc428a423a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xffffffff, 0x0, 0x19, 0xc440a1d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xc428a422a0, 0xc428a423c0, 0xc440a1d878, 0x558012, 0x588a5760, 0x3efb268)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xed01c4e60, 0x0, 0x0, 0xbdbe9)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xc428a422a0, 0xc428a42400, 0xa91b40, 0xc428a424a0, 0xc428a422a0, 0xc428a424a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xc420246100, 0xc43c501260, 0xc440a1da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43d176840, 0xa74a80, 0xc428ccd620, 0xc428a422a0, 0xc428a42360, 0x9464, 0xc420313b18, 0x0, 0xc420313b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43d176840, 0xc4415b5b60, 0xc420179d60, 0x0, 0x0, 0xc42ad71fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0700, 0xc420331380, 0x10265c0, 0xc43d176870, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0700, 0xc43d176870)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4204f0700)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 5328 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc420489c04)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc420489c00)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0xd9c46c7e266cb262)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc4435d9140, 0xc422334c2d, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4435d8c90, 0xc4435d9140, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4435d8c90, 0xc429a08a80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4435d8c90, 0xc42b396440, 0x0, 0xc4435da590, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4435d8c90, 0xa74a80, 0xc42b396440, 0x7fad6035c8e0, 0xc4435d8c90, 0x1038500, 0xc4435d8c60)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4435d8c60, 0xa74a80, 0xc42b396440, 0xc429a08500, 0xc429a08520, 0x7fad6035c8e0, 0xc4435d8c60, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4435d8c60, 0xa74a80, 0xc42b396440, 0x7fad6035c8e0, 0xc4435d8c60, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4435d8c60, 0xa74a80, 0xc42b396440, 0xc429a08500, 0xc429a08540, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xffffffff, 0x0, 0x19, 0xc42d70b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xc429a08500, 0xc429a08560, 0xc42d70b878, 0x558012, 0x588a575f, 0x39ef50b3)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xed01c4e5f, 0x0, 0x0, 0x121526)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xc429a08500, 0xc429a085c0, 0xa91b40, 0xc429a085e0, 0xc429a08500, 0xc429a085e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xc420246100, 0xc443746500, 0xc42d70ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4437407e0, 0xa74a80, 0xc42b396440, 0xc429a08500, 0xc429a08520, 0xc307, 0xc420e1ab18, 0x0, 0xc420e1ab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4437407e0, 0xc43bba52c0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb4a00, 0xc420331380, 0x10265c0, 0xc4437409c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb4a00, 0xc4437409c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cb4a00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8110 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44151f710, 0xc44151f740, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44151f710, 0xc438f47a40, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44151f710, 0xc442e267e0, 0x0, 0xc434142860, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44151f710, 0xa74a80, 0xc442e267e0, 0x7fad6035c8e0, 0xc44151f710, 0x1038500, 0xc44151f6e0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44151f6e0, 0xa74a80, 0xc442e267e0, 0xc438f47900, 0xc438f47920, 0x7fad6035c8e0, 0xc44151f6e0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151f6e0, 0xa74a80, 0xc442e267e0, 0x7fad6035c8e0, 0xc44151f6e0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44151f6e0, 0xa74a80, 0xc442e267e0, 0xc438f47900, 0xc438f47940, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xffffffff, 0x0, 0x19, 0xc42a6dd7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xc438f47900, 0xc438f47980, 0xc42a6dd878, 0x558012, 0x588a577a, 0xac79db1)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xed01c4e7a, 0x0, 0x0, 0x61ea4)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xc438f47900, 0xc438f479a0, 0xa91b40, 0xc438f479c0, 0xc438f47900, 0xc438f479c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xc420246100, 0xc43f0e50a0, 0xc42a6dda70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44151e390, 0xa74a80, 0xc442e267e0, 0xc438f47900, 0xc438f47920, 0x3650, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44151e390, 0xc4234bbbc0, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa700, 0xc420331380, 0x10265c0, 0xc44151e450, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa700, 0xc44151e450)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aa700)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9884 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437c0ced0, 0xc437c0cf30, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437c0ced0, 0xc43d1c4840, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437c0ced0, 0xc441054180, 0x0, 0xc438b75ae0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437c0ced0, 0xa74a80, 0xc441054180, 0x7fad6035c8e0, 0xc437c0ced0, 0x1038500, 0xc437c0ce70)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc437c0ce70, 0xa74a80, 0xc441054180, 0xc43d1c4720, 0xc43d1c4740, 0x7fad6035c8e0, 0xc437c0ce70, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c0ce70, 0xa74a80, 0xc441054180, 0x7fad6035c8e0, 0xc437c0ce70, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc437c0ce70, 0xa74a80, 0xc441054180, 0xc43d1c4720, 0xc43d1c4760, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xffffffff, 0x0, 0x19, 0xc4414557e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xc43d1c4720, 0xc43d1c4780, 0xc441455878, 0x558012, 0x588a588b, 0x335a74ca)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xed01c4f8b, 0x0, 0x0, 0x12a002)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xc43d1c4720, 0xc43d1c47a0, 0xa91b40, 0xc43d1c47c0, 0xc43d1c4720, 0xc43d1c47c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xc420246100, 0xc438b97360, 0xc441455a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc437c59dd0, 0xa74a80, 0xc441054180, 0xc43d1c4720, 0xc43d1c4740, 0x36b6, 0xc42023bb18, 0x0, 0xc42023bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc437c59dd0, 0xc4411972c0, 0xc420179d60, 0x0, 0x0, 0xc439252d40, 0x646e696b)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa300, 0xc420331380, 0x10265c0, 0xc437c59e00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa300, 0xc437c59e00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8886 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42210be00, 0xc42210be30, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42210be00, 0xc4432225c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42210be00, 0xc44a9cf980, 0x0, 0xc4223f7608, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42210be00, 0xa74a80, 0xc44a9cf980, 0x7fad6035c8e0, 0xc42210be00, 0x1038500, 0xc42210bdd0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42210bdd0, 0xa74a80, 0xc44a9cf980, 0xc4432224c0, 0xc4432224e0, 0x7fad6035c8e0, 0xc42210bdd0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210bdd0, 0xa74a80, 0xc44a9cf980, 0x7fad6035c8e0, 0xc42210bdd0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42210bdd0, 0xa74a80, 0xc44a9cf980, 0xc4432224c0, 0xc443222500, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xffffffff, 0x0, 0x19, 0xc4215bb7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xc4432224c0, 0xc443222520, 0xc4215bb878, 0x558012, 0x588a57f2, 0xf03c277)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xed01c4ef2, 0x0, 0x0, 0x6edfc)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xc4432224c0, 0xc443222540, 0xa91b40, 0xc443222560, 0xc4432224c0, 0xc443222560)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xc420246100, 0xc444b25ba0, 0xc4215bba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc428928150, 0xa74a80, 0xc44a9cf980, 0xc4432224c0, 0xc4432224e0, 0x8be8, 0xc42023db18, 0x0, 0xc42023db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc428928150, 0xc44ac1f3e0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41eab1840)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439c2a200, 0xc420331380, 0x10265c0, 0xc428928570, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439c2a200, 0xc428928570)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439c2a200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9885 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437b7fbf0, 0xc437b7fc20, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437b7fbf0, 0xc449234500, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437b7fbf0, 0xc43d1c5340, 0x0, 0xc438b0ab20, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437b7fbf0, 0xa74a80, 0xc43d1c5340, 0x7fad6035c8e0, 0xc437b7fbf0, 0x1038500, 0xc437b7eb40)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc437b7eb40, 0xa74a80, 0xc43d1c5340, 0xc449234400, 0xc449234420, 0x7fad6035c8e0, 0xc437b7eb40, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437b7eb40, 0xa74a80, 0xc43d1c5340, 0x7fad6035c8e0, 0xc437b7eb40, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc437b7eb40, 0xa74a80, 0xc43d1c5340, 0xc449234400, 0xc449234440, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xffffffff, 0x0, 0x19, 0xc4414577e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xc449234400, 0xc449234460, 0xc441457878, 0x558012, 0x588a588b, 0x374a0585)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xed01c4f8b, 0x0, 0x0, 0x65b8f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xc449234400, 0xc449234480, 0xa91b40, 0xc4492344a0, 0xc449234400, 0xc4492344a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xc420246100, 0xc438b75b90, 0xc441457a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc437c00c90, 0xa74a80, 0xc43d1c5340, 0xc449234400, 0xc449234420, 0x2b59, 0xc42023bb18, 0x0, 0xc42023bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc437c00c90, 0xc4411973e0, 0xc420179d60, 0x0, 0x0, 0xc439252d40, 0x646e696b)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa700, 0xc420331380, 0x10265c0, 0xc437c00cf0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa700, 0xc437c00cf0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438bfa700)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8112 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc446b12990, 0xc446b129c0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc446b12990, 0xc438de2d00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc446b12990, 0xc438f22880, 0x0, 0xc446b14740, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc446b12990, 0xa74a80, 0xc438f22880, 0x7fad6035c8e0, 0xc446b12990, 0x1038500, 0xc446b12960)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc446b12960, 0xa74a80, 0xc438f22880, 0xc438de2ba0, 0xc438de2bc0, 0x7fad6035c8e0, 0xc446b12960, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc446b12960, 0xa74a80, 0xc438f22880, 0x7fad6035c8e0, 0xc446b12960, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc446b12960, 0xa74a80, 0xc438f22880, 0xc438de2ba0, 0xc438de2be0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xffffffff, 0x0, 0x19, 0xc43ee437e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xc438de2ba0, 0xc438de2c20, 0xc43ee43878, 0x558012, 0x588a577a, 0xcbdef85)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xed01c4e7a, 0x0, 0x0, 0x4c792)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xc438de2ba0, 0xc438de2c40, 0xa91b40, 0xc438de2c60, 0xc438de2ba0, 0xc438de2c60)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xc420246100, 0xc4341429a0, 0xc43ee43a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43414e7b0, 0xa74a80, 0xc438f22880, 0xc438de2ba0, 0xc438de2bc0, 0x3da4, 0xc420044b18, 0x0, 0xc420044b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43414e7b0, 0xc4234bbe60, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aaf00, 0xc420331380, 0x10265c0, 0xc43414e870, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aaf00, 0xc43414e870)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aaf00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8153 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc42c13376c)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc42c133760)
	/usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc42c133760, 0xc42c212900)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e

goroutine 8562 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42244e540, 0xc42244e570, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42244e540, 0xc433c78a00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42244e540, 0xc433cb4160, 0x0, 0xc422165558, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42244e540, 0xa74a80, 0xc433cb4160, 0x7fad6035c8e0, 0xc42244e540, 0x1038500, 0xc42244e510)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42244e510, 0xa74a80, 0xc433cb4160, 0xc433c788e0, 0xc433c78900, 0x7fad6035c8e0, 0xc42244e510, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42244e510, 0xa74a80, 0xc433cb4160, 0x7fad6035c8e0, 0xc42244e510, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42244e510, 0xa74a80, 0xc433cb4160, 0xc433c788e0, 0xc433c78920, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xffffffff, 0x0, 0x19, 0xc448c317e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xc433c788e0, 0xc433c78960, 0xc448c31878, 0x558012, 0x588a57b9, 0x38658113)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xed01c4eb9, 0x0, 0x0, 0x19f243)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xc433c788e0, 0xc433c78980, 0xa91b40, 0xc433c789a0, 0xc433c788e0, 0xc433c789a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xc420246100, 0xc422ecc2e0, 0xc448c31a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc423344420, 0xa74a80, 0xc433cb4160, 0xc433c788e0, 0xc433c78900, 0x87ff, 0xc420319b18, 0x0, 0xc420319b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc423344420, 0xc4235d4cc0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f5d1149)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd200, 0xc420331380, 0x10265c0, 0xc423344540, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd200, 0xc423344540)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 5364 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204962c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204962c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0xe9d56397ab2f3733)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc441195980, 0xc422334c2d, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441195920, 0xc441195980, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441195920, 0xc429094fc0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441195920, 0xc4292ec3c0, 0x0, 0xc44106b3e0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441195920, 0xa74a80, 0xc4292ec3c0, 0x7fad6035c8e0, 0xc441195920, 0x1038500, 0xc4411958f0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4411958f0, 0xa74a80, 0xc4292ec3c0, 0xc429094c80, 0xc429094cc0, 0x7fad6035c8e0, 0xc4411958f0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4411958f0, 0xa74a80, 0xc4292ec3c0, 0x7fad6035c8e0, 0xc4411958f0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4411958f0, 0xa74a80, 0xc4292ec3c0, 0xc429094c80, 0xc429094ce0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xffffffff, 0x0, 0x19, 0xc42a6e17e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xc429094c80, 0xc429094d40, 0xc42a6e1878, 0x558012, 0x588a5760, 0x26c9b7f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xed01c4e60, 0x0, 0x0, 0x178f91)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xc429094c80, 0xc429094e20, 0xa91b40, 0xc429094e60, 0xc429094c80, 0xc429094e60)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xc420246100, 0xc441865630, 0xc42a6e1a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc44132b5c0, 0xa74a80, 0xc4292ec3c0, 0xc429094c80, 0xc429094cc0, 0x961e, 0xc43d5d2b18, 0x0, 0xc43d5d2b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc44132b5c0, 0xc440871b60, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cad400, 0xc420331380, 0x10265c0, 0xc44132b5f0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cad400, 0xc44132b5f0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc420cad400)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9888 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437885110, 0xc437885140, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437885110, 0xc43ce23480, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437885110, 0xc44748ea60, 0x0, 0xc43897fc30, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437885110, 0xa74a80, 0xc44748ea60, 0x7fad6035c8e0, 0xc437885110, 0x1038500, 0xc4378850e0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4378850e0, 0xa74a80, 0xc44748ea60, 0xc43ce23380, 0xc43ce233a0, 0x7fad6035c8e0, 0xc4378850e0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4378850e0, 0xa74a80, 0xc44748ea60, 0x7fad6035c8e0, 0xc4378850e0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4378850e0, 0xa74a80, 0xc44748ea60, 0xc43ce23380, 0xc43ce233c0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0xffffffff, 0x0, 0x19, 0xc43d89b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0xc43ce23380, 0xc43ce233e0, 0xc43d89b878, 0x558012, 0x588a588b, 0x3a1d1363)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0x41167e, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0xc43ce23380, 0xc43ce23400, 0xa91b40, 0xc43ce23420, 0xc43ce23380, 0xc43ce23420)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0xc420246100, 0xc4389d41c0, 0xc43d89ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4379569c0, 0xa74a80, 0xc44748ea60, 0xc43ce23380, 0xc43ce233a0, 0x5743, 0xc420044b18, 0x0, 0xc420044b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4379569c0, 0xc43c651bc0, 0xc420179d60, 0x0, 0x0, 0xc439252d40, 0x646e696b)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438bfb900, 0xc420331380, 0x10265c0, 0xc437956a20, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438bfb900, 0xc437956a20)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438bfb900)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8267 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).numUsers(0xc4201e3078, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:68 +0x3a
github.com/weaveworks/cortex/ingester.(*Ingester).Collect(0xc4201e3040, 0xc446b0daa0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:563 +0x4e
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2(0xc444b24d40, 0xc446b0daa0, 0x10343c0, 0xc4201e3040)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:433 +0x63
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:434 +0x326

goroutine 5580 [select, 13 minutes]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc42000d8c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 11429 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc431de214c)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc431de2140)
	/usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc431de2140, 0xc43c3da7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e

goroutine 11601 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43b3fd8f0, 0xc43b3fd920, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43b3fd8f0, 0xc43cf35c80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43b3fd8f0, 0xc43d0ea740, 0x0, 0xc42ff18a98, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43b3fd8f0, 0xa74a80, 0xc43d0ea740, 0x7fad6035c8e0, 0xc43b3fd8f0, 0x1038500, 0xc43b3fd8c0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43b3fd8c0, 0xa74a80, 0xc43d0ea740, 0xc43cf35b80, 0xc43cf35ba0, 0x7fad6035c8e0, 0xc43b3fd8c0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43b3fd8c0, 0xa74a80, 0xc43d0ea740, 0x7fad6035c8e0, 0xc43b3fd8c0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43b3fd8c0, 0xa74a80, 0xc43d0ea740, 0xc43cf35b80, 0xc43cf35bc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xffffffff, 0x0, 0x19, 0xc43b8337e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xc43cf35b80, 0xc43cf35be0, 0xc43b833878, 0x558012, 0x588a5996, 0x105d0346)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xed01c5096, 0x0, 0x0, 0xb6f82)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xc43cf35b80, 0xc43cf35c00, 0xa91b40, 0xc43cf35c20, 0xc43cf35b80, 0xc43cf35c20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xc420246100, 0xc430195890, 0xc43b833a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc441b0d350, 0xa74a80, 0xc43d0ea740, 0xc43cf35b80, 0xc43cf35ba0, 0x7c74, 0xc420e1cb18, 0x0, 0xc420e1cb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc441b0d350, 0xc422873740, 0xc420179d60, 0x0, 0x0, 0x44c49082079f0910, 0x9de395f9d710400f)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438065700, 0xc420331380, 0x10265c0, 0xc441b0d380, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438065700, 0xc441b0d380)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438065700)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10662 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43130bdd0, 0xc43130be00, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43130bdd0, 0xc444079100, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43130bdd0, 0xc4483f0e20, 0x0, 0xc43557ed50, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43130bdd0, 0xa74a80, 0xc4483f0e20, 0x7fad6035c8e0, 0xc43130bdd0, 0x1038500, 0xc43130bda0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43130bda0, 0xa74a80, 0xc4483f0e20, 0xc444079000, 0xc444079020, 0x7fad6035c8e0, 0xc43130bda0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43130bda0, 0xa74a80, 0xc4483f0e20, 0x7fad6035c8e0, 0xc43130bda0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43130bda0, 0xa74a80, 0xc4483f0e20, 0xc444079000, 0xc444079040, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xffffffff, 0x0, 0x19, 0xc4472157e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xc444079000, 0xc444079060, 0xc447215878, 0x558012, 0x588a5921, 0x3a91d9bb)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xed01c5021, 0x0, 0x0, 0x104e94)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xc444079000, 0xc444079080, 0xa91b40, 0xc4440790a0, 0xc444079000, 0xc4440790a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xc420246100, 0xc435730ff0, 0xc447215a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43138af30, 0xa74a80, 0xc4483f0e20, 0xc444079000, 0xc444079020, 0x88ec, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43138af30, 0xc436c75320, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41edcc21c)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065400, 0xc420331380, 0x10265c0, 0xc43138b080, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065400, 0xc43138b080)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065400)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10659 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4311cd3b0, 0xc4311cd410, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4311cd3b0, 0xc44973b420, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4311cd3b0, 0xc4438e8800, 0x0, 0xc4368f5478, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4311cd3b0, 0xa74a80, 0xc4438e8800, 0x7fad6035c8e0, 0xc4311cd3b0, 0x1038500, 0xc4311cd350)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4311cd350, 0xa74a80, 0xc4438e8800, 0xc43dd3bca0, 0xc43dd3bdc0, 0x7fad6035c8e0, 0xc4311cd350, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4311cd350, 0xa74a80, 0xc4438e8800, 0x7fad6035c8e0, 0xc4311cd350, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4311cd350, 0xa74a80, 0xc4438e8800, 0xc43dd3bca0, 0xc43dd3bec0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xffffffff, 0x0, 0x19, 0xc4322057e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xc43dd3bca0, 0xc43dd3bfc0, 0xc432205878, 0x558012, 0x588a5900, 0xf9d91e7)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xc432205900, 0x0, 0x0, 0x71)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xc43dd3bca0, 0xc44973a040, 0xa91b40, 0xc44973ae20, 0xc43dd3bca0, 0xc44973ae20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xc420246100, 0xc436a82190, 0xc432205a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42da76660, 0xa74a80, 0xc4438e8800, 0xc43dd3bca0, 0xc43dd3bdc0, 0x5617, 0xc420314b18, 0x0, 0xc420314b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42da76660, 0xc436014a20, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f0f4464)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390faf00, 0xc420331380, 0x10265c0, 0xc42da766c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390faf00, 0xc42da766c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390faf00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8419 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc42b5b01b0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f
github.com/weaveworks/cortex/ingester.(*Ingester).LabelValues(0xc4201e3040, 0x7fad6035c8e0, 0xc42b5b01b0, 0xc42c6276f0, 0x0, 0xc42c627718, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:298 +0x60
github.com/weaveworks/cortex._Ingester_LabelValues_Handler.func1(0x7fad6035c8e0, 0xc42b5b01b0, 0xa73d60, 0xc42c6276f0, 0x7fad6035c8e0, 0xc42b5b01b0, 0x1038500, 0xc42b5b0180)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:457 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42b5b0180, 0xa73d60, 0xc42c6276f0, 0xc434fcff60, 0xc434fcff80, 0x7fad6035c8e0, 0xc42b5b0180, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b5b0180, 0xa73d60, 0xc42c6276f0, 0x7fad6035c8e0, 0xc42b5b0180, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42b5b0180, 0xa73d60, 0xc42c6276f0, 0xc434fcff60, 0xc434fcffa0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xffffffff, 0x0, 0x19, 0xc4427e57e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xc434fcff60, 0xc434fcffc0, 0xc4427e5878, 0x558012, 0x588a57a4, 0x1650ef3a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xc4427e5900, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xc434fcff60, 0xc434fcffe0, 0xa91b40, 0xc434fcc000, 0xc434fcff60, 0xc434fcc000)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xc420246100, 0xc42c6276e0, 0xc4427e5a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42b5b0060, 0xa73d60, 0xc42c6276f0, 0xc434fcff60, 0xc434fcff80, 0xa, 0xc420e1bb18, 0x0, 0xc420e1bb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_LabelValues_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42b5b0060, 0xc42b7fa8a0, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:459 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc42646d600, 0xc420331380, 0x10265f0, 0xc42b5b0120, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc42646d600, 0xc42b5b0120)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc43c1fd850, 0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc42646d600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 11986 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43e5ec630, 0xc43e5ec660, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43e5ec630, 0xc4484c4c80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43e5ec630, 0xc44334b500, 0x0, 0xc432a37310, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43e5ec630, 0xa74a80, 0xc44334b500, 0x7fad6035c8e0, 0xc43e5ec630, 0x1038500, 0xc43e5ec600)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43e5ec600, 0xa74a80, 0xc44334b500, 0xc4484c4b80, 0xc4484c4ba0, 0x7fad6035c8e0, 0xc43e5ec600, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43e5ec600, 0xa74a80, 0xc44334b500, 0x7fad6035c8e0, 0xc43e5ec600, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43e5ec600, 0xa74a80, 0xc44334b500, 0xc4484c4b80, 0xc4484c4bc0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xffffffff, 0x0, 0x19, 0xc4385a17e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xc4484c4b80, 0xc4484c4be0, 0xc4385a1878, 0x558012, 0x588a59d2, 0x1224f2f3)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xed01c50d2, 0x0, 0x0, 0x185d83)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xc4484c4b80, 0xc4484c4c00, 0xa91b40, 0xc4484c4c20, 0xc4484c4b80, 0xc4484c4c20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xc420246100, 0xc434f81cc0, 0xc4385a1a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442452480, 0xa74a80, 0xc44334b500, 0xc4484c4b80, 0xc4484c4ba0, 0x7c68, 0xc420239b18, 0x0, 0xc420239b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442452480, 0xc43e3076e0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6200, 0xc420331380, 0x10265c0, 0xc4424524b0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6200, 0xc4424524b0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4396a6200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8650 [chan send, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc430b3e4b0, 0xc436c445a0, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 9664 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44603fef0, 0xc44603ff20, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44603fef0, 0xc449db44a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44603fef0, 0xc43c25a640, 0x0, 0xc441e558d0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44603fef0, 0xa74a80, 0xc43c25a640, 0x7fad6035c8e0, 0xc44603fef0, 0x1038500, 0xc44603fec0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44603fec0, 0xa74a80, 0xc43c25a640, 0xc449db98a0, 0xc449db9a20, 0x7fad6035c8e0, 0xc44603fec0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44603fec0, 0xa74a80, 0xc43c25a640, 0x7fad6035c8e0, 0xc44603fec0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44603fec0, 0xa74a80, 0xc43c25a640, 0xc449db98a0, 0xc449db9ba0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xffffffff, 0x0, 0x19, 0xc42eb3d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xc449db98a0, 0xc449db9d40, 0xc42eb3d878, 0x558012, 0x588a586a, 0x103547fa)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xc42eb3d900, 0x0, 0x0, 0x1)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xc449db98a0, 0xc449db9ea0, 0xa91b40, 0xc449db4020, 0xc449db98a0, 0xc449db4020)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xc420246100, 0xc43ff6bf40, 0xc42eb3da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43dff1050, 0xa74a80, 0xc43c25a640, 0xc449db98a0, 0xc449db9a20, 0x7de0, 0xc420e1ab18, 0x0, 0xc420e1ab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43dff1050, 0xc445cf84e0, 0xc420179d60, 0x0, 0x0, 0x7fad625dc000, 0xc41eac8542)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87a00, 0xc420331380, 0x10265c0, 0xc43dff1080, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87a00, 0xc43dff1080)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc439b87a00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8209 [chan send, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc4200d4a50, 0xc446b0da40, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 8150 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42c1daf30, 0xc42c1daf60, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42c1daf30, 0xc43a5a11c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42c1daf30, 0xc43a3bce60, 0x0, 0xc42c133610, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42c1daf30, 0xa74a80, 0xc43a3bce60, 0x7fad6035c8e0, 0xc42c1daf30, 0x1038500, 0xc42c1daf00)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42c1daf00, 0xa74a80, 0xc43a3bce60, 0xc43a5a10c0, 0xc43a5a10e0, 0x7fad6035c8e0, 0xc42c1daf00, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42c1daf00, 0xa74a80, 0xc43a3bce60, 0x7fad6035c8e0, 0xc42c1daf00, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42c1daf00, 0xa74a80, 0xc43a3bce60, 0xc43a5a10c0, 0xc43a5a1100, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xffffffff, 0x0, 0x19, 0xc439edf7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xc43a5a10c0, 0xc43a5a1120, 0xc439edf878, 0x558012, 0x588a579c, 0x3dd1ed7)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xed01c4e9c, 0x0, 0x0, 0x88d4b)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xc43a5a10c0, 0xc43a5a1140, 0xa91b40, 0xc43a5a1160, 0xc43a5a10c0, 0xc43a5a1160)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xc420246100, 0xc42d4e0990, 0xc439edfa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d4c2e70, 0xa74a80, 0xc43a3bce60, 0xc43a5a10c0, 0xc43a5a10e0, 0x73f7, 0xc42023cb18, 0x0, 0xc42023cb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d4c2e70, 0xc44088fd40, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42646c800, 0xc420331380, 0x10265c0, 0xc42d4c2ea0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42646c800, 0xc42d4c2ea0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc42646c800)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 9880 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc437eb4360, 0xc437eb4390, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc437eb4360, 0xc441109040, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc437eb4360, 0xc44311f6e0, 0x0, 0xc438d77770, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc437eb4360, 0xa74a80, 0xc44311f6e0, 0x7fad6035c8e0, 0xc437eb4360, 0x1038500, 0xc437eb4300)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc437eb4300, 0xa74a80, 0xc44311f6e0, 0xc441108f20, 0xc441108f40, 0x7fad6035c8e0, 0xc437eb4300, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc437eb4300, 0xa74a80, 0xc44311f6e0, 0x7fad6035c8e0, 0xc437eb4300, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc437eb4300, 0xa74a80, 0xc44311f6e0, 0xc441108f20, 0xc441108f60, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xffffffff, 0x0, 0x19, 0xc44578d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xc441108f20, 0xc441108fa0, 0xc44578d878, 0x558012, 0x588a5888, 0x103f7fdd)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xc44578d900, 0x0, 0x0, 0x7f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xc441108f20, 0xc441108fc0, 0xa91b40, 0xc441108fe0, 0xc441108f20, 0xc441108fe0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xc420246100, 0xc438e98fd0, 0xc44578da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc438262ff0, 0xa74a80, 0xc44311f6e0, 0xc441108f20, 0xc441108f40, 0x7b93, 0xc420319b18, 0x0, 0xc420319b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc438262ff0, 0xc4415b5560, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f6f34f0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b4800, 0xc420331380, 0x10265c0, 0xc438263020, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b4800, 0xc438263020)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4393b4800)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8516 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4223f5860, 0xc4223f5890, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4223f5860, 0xc433bc11c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4223f5860, 0xc434049240, 0x0, 0xc4223f7c20, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4223f5860, 0xa74a80, 0xc434049240, 0x7fad6035c8e0, 0xc4223f5860, 0x1038500, 0xc4223f5830)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4223f5830, 0xa74a80, 0xc434049240, 0xc433bc10a0, 0xc433bc10c0, 0x7fad6035c8e0, 0xc4223f5830, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4223f5830, 0xa74a80, 0xc434049240, 0x7fad6035c8e0, 0xc4223f5830, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4223f5830, 0xa74a80, 0xc434049240, 0xc433bc10a0, 0xc433bc10e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xffffffff, 0x0, 0x19, 0xc43fccf7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xc433bc10a0, 0xc433bc1100, 0xc43fccf878, 0x558012, 0x588a57b9, 0x3a0b8463)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xed01c4eb9, 0x0, 0x0, 0x4b18ff)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xc433bc10a0, 0xc433bc1120, 0xa91b40, 0xc433bc1140, 0xc433bc10a0, 0xc433bc1140)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xc420246100, 0xc426a91790, 0xc43fccfa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4269fd860, 0xa74a80, 0xc434049240, 0xc433bc10a0, 0xc433bc10c0, 0x63b9, 0xc420e18b18, 0x0, 0xc420e18b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4269fd860, 0xc428757440, 0xc420179d60, 0x0, 0x0, 0x4d1279726f746973, 0x6f642e7865646e69)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4ec00, 0xc420331380, 0x10265c0, 0xc4269fd980, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4ec00, 0xc4269fd980)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc422b4ec00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8081 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc422f9ec90, 0xc422f9ecc0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc422f9ec90, 0xc433cc2fe0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc422f9ec90, 0xc433d357c0, 0x0, 0xc422ecc290, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc422f9ec90, 0xa74a80, 0xc433d357c0, 0x7fad6035c8e0, 0xc422f9ec90, 0x1038500, 0xc422f9ec60)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc422f9ec60, 0xa74a80, 0xc433d357c0, 0xc433cc2ea0, 0xc433cc2ec0, 0x7fad6035c8e0, 0xc422f9ec60, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc422f9ec60, 0xa74a80, 0xc433d357c0, 0x7fad6035c8e0, 0xc422f9ec60, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc422f9ec60, 0xa74a80, 0xc433d357c0, 0xc433cc2ea0, 0xc433cc2ee0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xffffffff, 0x0, 0x19, 0xc448f037e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xc433cc2ea0, 0xc433cc2f20, 0xc448f03878, 0x558012, 0x588a57b9, 0x38574e00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xed01c4eb9, 0x0, 0x0, 0xa931e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xc433cc2ea0, 0xc433cc2f40, 0xa91b40, 0xc433cc2f60, 0xc433cc2ea0, 0xc433cc2f60)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xc420246100, 0xc423602b60, 0xc448f03a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc423344180, 0xa74a80, 0xc433d357c0, 0xc433cc2ea0, 0xc433cc2ec0, 0x92b3, 0xc42023cb18, 0x0, 0xc42023cb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc423344180, 0xc4235d41e0, 0xc420179d60, 0x0, 0x0, 0x61632f736f6d6564, 0x36383639653a7472)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd100, 0xc420331380, 0x10265c0, 0xc4233445a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd100, 0xc4233445a0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4228bd100)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10274 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4359e5bf0, 0xc4359e5c20, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4359e5bf0, 0xc440825440, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4359e5bf0, 0xc43a753600, 0x0, 0xc4372372a0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4359e5bf0, 0xa74a80, 0xc43a753600, 0x7fad6035c8e0, 0xc4359e5bf0, 0x1038500, 0xc4359e5bc0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4359e5bc0, 0xa74a80, 0xc43a753600, 0xc440825320, 0xc440825340, 0x7fad6035c8e0, 0xc4359e5bc0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4359e5bc0, 0xa74a80, 0xc43a753600, 0x7fad6035c8e0, 0xc4359e5bc0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4359e5bc0, 0xa74a80, 0xc43a753600, 0xc440825320, 0xc440825360, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xffffffff, 0x0, 0x19, 0xc4422737e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xc440825320, 0xc440825380, 0xc442273878, 0x558012, 0x588a58c7, 0x3728a266)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xed01c4fc7, 0x0, 0x0, 0x11d9b1)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xc440825320, 0xc4408253c0, 0xa91b40, 0xc4408253e0, 0xc440825320, 0xc4408253e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xc420246100, 0xc4372de5c0, 0xc442273a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc435b26780, 0xa74a80, 0xc43a753600, 0xc440825320, 0xc440825340, 0x4899, 0xc420318b18, 0x0, 0xc420318b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc435b26780, 0xc4320ccb40, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc200, 0xc420331380, 0x10265c0, 0xc435b267e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc200, 0xc435b267e0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8651 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4386531dc)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc4386531d0)
	/usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc4386531d0, 0xc436c44600)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e

goroutine 8721 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc442ded230, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc442ded230, 0x159d0153bd2, 0x159d04d14b2, 0xc42376afe8, 0x1, 0x1, 0xc442ded200, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:272 +0x7e
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc442ded230, 0xc442ded170, 0x0, 0xc442ca6600, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc442ded230, 0xa934c0, 0xc442ded170, 0x7fad6035c8e0, 0xc442ded230, 0x1038500, 0xc442ded200)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc442ded200, 0xa934c0, 0xc442ded170, 0xc432045800, 0xc432045820, 0x7fad6035c8e0, 0xc442ded200, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded200, 0xa934c0, 0xc442ded170, 0x7fad6035c8e0, 0xc442ded200, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc442ded200, 0xa934c0, 0xc442ded170, 0xc432045800, 0xc432045860, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xffffffff, 0x0, 0x19, 0xc431f097e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xc432045800, 0xc4320458a0, 0xc431f09878, 0x558012, 0x588a57df, 0x1570a41a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xed01c4edf, 0x0, 0x0, 0x9743)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xc432045800, 0xc4320458c0, 0xa91b40, 0xc432045900, 0xc432045800, 0xc432045900)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xc420246100, 0xc442ca65c0, 0xc431f09a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442ded050, 0xa934c0, 0xc442ded170, 0xc432045800, 0xc432045820, 0x41, 0xc420e1ab18, 0x0, 0xc420e1ab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442ded050, 0xc43ac10f60, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc420cb9e00, 0xc420331380, 0x10265d8, 0xc442ded110, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc420cb9e00, 0xc442ded110)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc43e3ca700, 0xc4202460c0, 0x103ac20, 0xc42000d8c0, 0xc420cb9e00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8148 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42751a5a0, 0xc42751a5d0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42751a5a0, 0xc437af2b80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42751a5a0, 0xc437b5e600, 0x0, 0xc4275185f8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42751a5a0, 0xa74a80, 0xc437b5e600, 0x7fad6035c8e0, 0xc42751a5a0, 0x1038500, 0xc42751a570)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42751a570, 0xa74a80, 0xc437b5e600, 0xc437af2a60, 0xc437af2a80, 0x7fad6035c8e0, 0xc42751a570, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42751a570, 0xa74a80, 0xc437b5e600, 0x7fad6035c8e0, 0xc42751a570, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42751a570, 0xa74a80, 0xc437b5e600, 0xc437af2a60, 0xc437af2aa0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xffffffff, 0x0, 0x19, 0xc4454c37e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xc437af2a60, 0xc437af2ac0, 0xc4454c3878, 0x558012, 0x588a579b, 0x36a0864d)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xed01c4e9b, 0x0, 0x0, 0x9d15b)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xc437af2a60, 0xc437af2ae0, 0xa91b40, 0xc437af2b00, 0xc437af2a60, 0xc437af2b00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xc420246100, 0xc4292d9b20, 0xc4454c3a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc4292dbef0, 0xa74a80, 0xc437b5e600, 0xc437af2a60, 0xc437af2a80, 0x6e61, 0xc420e1db18, 0x0, 0xc420e1db08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc4292dbef0, 0xc44088ec00, 0xc420179d60, 0x0, 0x0, 0x6c616e7265746e69, 0x412656c020a0a0a)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c200, 0xc420331380, 0x10265c0, 0xc4292dbf20, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c200, 0xc4292dbf20)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc42646c200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10096 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d9a17d0, 0xc42d9a1800, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d9a17d0, 0xc441ed7a60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d9a17d0, 0xc442fa89c0, 0x0, 0xc436a82098, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d9a17d0, 0xa74a80, 0xc442fa89c0, 0x7fad6035c8e0, 0xc42d9a17d0, 0x1038500, 0xc42d9a17a0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d9a17a0, 0xa74a80, 0xc442fa89c0, 0xc441ed7920, 0xc441ed7960, 0x7fad6035c8e0, 0xc42d9a17a0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d9a17a0, 0xa74a80, 0xc442fa89c0, 0x7fad6035c8e0, 0xc42d9a17a0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d9a17a0, 0xa74a80, 0xc442fa89c0, 0xc441ed7920, 0xc441ed7980, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xffffffff, 0x0, 0x19, 0xc432f7d7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xc441ed7920, 0xc441ed79a0, 0xc432f7d878, 0x558012, 0x588a5900, 0xf82dd5f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xed01c5000, 0x0, 0x0, 0x1765b3)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xc441ed7920, 0xc441ed79c0, 0xa91b40, 0xc441ed79e0, 0xc441ed7920, 0xc441ed79e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xc420246100, 0xc436b5cf30, 0xc432f7da70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42cd609f0, 0xa74a80, 0xc442fa89c0, 0xc441ed7920, 0xc441ed7960, 0x51d9, 0xc420314b18, 0x0, 0xc420314b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42cd609f0, 0xc436014180, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f0f4464)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fa500, 0xc420331380, 0x10265c0, 0xc42cd60a50, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fa500, 0xc42cd60a50)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4390fa500)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8802 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc442ded560, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f
github.com/weaveworks/cortex/ingester.(*Ingester).LabelValues(0xc4201e3040, 0x7fad6035c8e0, 0xc442ded560, 0xc442ca66e0, 0x0, 0xc442ca6708, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:298 +0x60
github.com/weaveworks/cortex._Ingester_LabelValues_Handler.func1(0x7fad6035c8e0, 0xc442ded560, 0xa73d60, 0xc442ca66e0, 0x7fad6035c8e0, 0xc442ded560, 0x1038500, 0xc442ded530)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:457 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc442ded530, 0xa73d60, 0xc442ca66e0, 0xc432045b00, 0xc432045b60, 0x7fad6035c8e0, 0xc442ded530, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded530, 0xa73d60, 0xc442ca66e0, 0x7fad6035c8e0, 0xc442ded530, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc442ded530, 0xa73d60, 0xc442ca66e0, 0xc432045b00, 0xc432045b80, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xffffffff, 0x0, 0x19, 0xc431f0f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xc432045b00, 0xc432045ba0, 0xc431f0f878, 0x558012, 0x588a57e0, 0x1e22dc5e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xed01c4ee0, 0x0, 0x0, 0xcc22)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xc432045b00, 0xc432045c00, 0xa91b40, 0xc432045c20, 0xc432045b00, 0xc432045c20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xc420246100, 0xc442ca66d0, 0xc431f0fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442ded410, 0xa73d60, 0xc442ca66e0, 0xc432045b00, 0xc432045b60, 0xa, 0xc420e1ab18, 0x0, 0xc420e1ab08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_LabelValues_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442ded410, 0xc43ac11080, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:459 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc420cac500, 0xc420331380, 0x10265f0, 0xc442ded4d0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc420cac500, 0xc442ded4d0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc440f5ebb0, 0xc4202460c0, 0x103ac20, 0xc42000d740, 0xc420cac500)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 11989 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc439a3be90, 0xc439a3bec0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc439a3be90, 0xc439187e00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc439a3be90, 0xc43e464d20, 0x0, 0xc42f41d450, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc439a3be90, 0xa74a80, 0xc43e464d20, 0x7fad6035c8e0, 0xc439a3be90, 0x1038500, 0xc439a3be60)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc439a3be60, 0xa74a80, 0xc43e464d20, 0xc439187ce0, 0xc439187d00, 0x7fad6035c8e0, 0xc439a3be60, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc439a3be60, 0xa74a80, 0xc43e464d20, 0x7fad6035c8e0, 0xc439a3be60, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc439a3be60, 0xa74a80, 0xc43e464d20, 0xc439187ce0, 0xc439187d40, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xffffffff, 0x0, 0x19, 0xc4376b37e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xc439187ce0, 0xc439187d60, 0xc4376b3878, 0x558012, 0x588a59d6, 0x9570e8)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xed01c50d6, 0x0, 0x0, 0xffe49)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xc439187ce0, 0xc439187d80, 0xa91b40, 0xc439187da0, 0xc439187ce0, 0xc439187da0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xc420246100, 0xc42f54d490, 0xc4376b3a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442a1cc30, 0xa74a80, 0xc43e464d20, 0xc439187ce0, 0xc439187d00, 0x6857, 0xc420e1fb18, 0x0, 0xc420e1fb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442a1cc30, 0xc44103cae0, 0xc420179d60, 0x0, 0x0, 0x3066732d36303736, 0x656c020a090a3479)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4394f3000, 0xc420331380, 0x10265c0, 0xc442a1cc60, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4394f3000, 0xc442a1cc60)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4394f3000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 11291 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42943ce70, 0xc42943cf30, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42943ce70, 0xc43dd1f740, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42943ce70, 0xc449760020, 0x0, 0xc432cf33b0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42943ce70, 0xa74a80, 0xc449760020, 0x7fad6035c8e0, 0xc42943ce70, 0x1038500, 0xc429465fb0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc429465fb0, 0xa74a80, 0xc449760020, 0xc43dd1f620, 0xc43dd1f640, 0x7fad6035c8e0, 0xc429465fb0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429465fb0, 0xa74a80, 0xc449760020, 0x7fad6035c8e0, 0xc429465fb0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc429465fb0, 0xa74a80, 0xc449760020, 0xc43dd1f620, 0xc43dd1f660, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xffffffff, 0x0, 0x19, 0xc437c7b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xc43dd1f620, 0xc43dd1f680, 0xc437c7b878, 0x558012, 0x588a5978, 0x12cd7b59)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xed01c5078, 0x0, 0x0, 0xb4e72)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xc43dd1f620, 0xc43dd1f6a0, 0xa91b40, 0xc43dd1f6e0, 0xc43dd1f620, 0xc43dd1f6e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xc420246100, 0xc432dd5da0, 0xc437c7ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a29a480, 0xa74a80, 0xc449760020, 0xc43dd1f620, 0xc43dd1f640, 0x79e2, 0xc420e18b18, 0x0, 0xc420e18b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a29a480, 0xc442bd43c0, 0xc420179d60, 0x0, 0x0, 0x6236633266303862, 0x3462316364333339)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa300, 0xc420331380, 0x10265c0, 0xc4298885d0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa300, 0xc4298885d0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 5579 [IO wait]:
net.runtime_pollWait(0x7fad6039cfa0, 0x72, 0xb)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc43d2bfc60, 0x72, 0xc4202388d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc43d2bfc60, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc43d2bfc00, 0xc43e3d2000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc43e50a018, 0xc43e3d2000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc445d837a0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc445d837a0, 0xc42000d838, 0x9, 0x9, 0x43fe80, 0xc42c6149f0, 0x0)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc445d837a0, 0xc42000d838, 0x9, 0x9, 0x9, 0xc420238ae8, 0x451a90, 0xc441066c80)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc445d837a0, 0xc42000d838, 0x9, 0x9, 0xc42c6149f0, 0x0, 0x1)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc42000d838, 0x9, 0x9, 0x102f380, 0xc445d837a0, 0x0, 0xc400000000, 0xc420238c80, 0x7f3105)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42000d800, 0xf, 0xf, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc43e509710, 0xc44283d9e0, 0xc44283d9e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42000d8c0, 0xc43e5097a0, 0xb475b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202460c0, 0x103ac20, 0xc42000d8c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202460c0, 0x103ae60, 0xc43e50a018, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202460c0, 0x103ae60, 0xc43e50a018)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 11600 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc441b0c000, 0xc441b0c030, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc441b0c000, 0xc43f89b960, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc441b0c000, 0xc43d343f80, 0x0, 0xc430195480, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc441b0c000, 0xa74a80, 0xc43d343f80, 0x7fad6035c8e0, 0xc441b0c000, 0x1038500, 0xc43cdedfb0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43cdedfb0, 0xa74a80, 0xc43d343f80, 0xc43f89b840, 0xc43f89b860, 0x7fad6035c8e0, 0xc43cdedfb0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43cdedfb0, 0xa74a80, 0xc43d343f80, 0x7fad6035c8e0, 0xc43cdedfb0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43cdedfb0, 0xa74a80, 0xc43d343f80, 0xc43f89b840, 0xc43f89b8a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xffffffff, 0x0, 0x19, 0xc4310cb7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xc43f89b840, 0xc43f89b8c0, 0xc4310cb878, 0x558012, 0x588a5996, 0x104a2b66)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xed01c5096, 0x0, 0x0, 0xc9d51)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xc43f89b840, 0xc43f89b8e0, 0xa91b40, 0xc43f89b900, 0xc43f89b840, 0xc43f89b900)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xc420246100, 0xc4303977e0, 0xc4310cba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc445745b00, 0xa74a80, 0xc43d343f80, 0xc43f89b840, 0xc43f89b860, 0x79a4, 0xc420e1cb18, 0x0, 0xc420e1cb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc445745b00, 0xc425c87a40, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ef31943)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065000, 0xc420331380, 0x10265c0, 0xc445745b30, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065000, 0xc445745b30)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438065000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8553 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4386526ec)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc4386526e0)
	/usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc4386526e0, 0xc42e9d8900)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e

goroutine 11216 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42aef1050, 0xc42aef10b0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42aef1050, 0xc4444f4080, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42aef1050, 0xc444da8a40, 0x0, 0xc43523fb50, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42aef1050, 0xa74a80, 0xc444da8a40, 0x7fad6035c8e0, 0xc42aef1050, 0x1038500, 0xc42aef0f90)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42aef0f90, 0xa74a80, 0xc444da8a40, 0xc42d41bf60, 0xc42d41bf80, 0x7fad6035c8e0, 0xc42aef0f90, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42aef0f90, 0xa74a80, 0xc444da8a40, 0x7fad6035c8e0, 0xc42aef0f90, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42aef0f90, 0xa74a80, 0xc444da8a40, 0xc42d41bf60, 0xc42d41bfa0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xffffffff, 0x0, 0x19, 0xc42805b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xc42d41bf60, 0xc42d41bfc0, 0xc42805b878, 0x558012, 0x588a595d, 0x3a9b5973)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xc42805b900, 0x0, 0x0, 0x1)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xc42d41bf60, 0xc42d41bfe0, 0xa91b40, 0xc4444f4020, 0xc42d41bf60, 0xc4444f4020)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xc420246100, 0xc4352bb640, 0xc42805ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42fd7c5a0, 0xa74a80, 0xc444da8a40, 0xc42d41bf60, 0xc42d41bf80, 0x6f18, 0xc440d64b18, 0x0, 0xc440d64b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42fd7c5a0, 0xc43dd27c80, 0xc420179d60, 0x0, 0x0, 0xc434e02fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7100, 0xc420331380, 0x10265c0, 0xc42fd7c600, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7100, 0xc42fd7c600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7100)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10273 [semacquire, 8 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc435ad2390, 0xc435ad23f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc435ad2390, 0xc43a752b80, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc435ad2390, 0xc43f2e5fc0, 0x0, 0xc4372de548, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc435ad2390, 0xa74a80, 0xc43f2e5fc0, 0x7fad6035c8e0, 0xc435ad2390, 0x1038500, 0xc435ad2330)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc435ad2330, 0xa74a80, 0xc43f2e5fc0, 0xc43a73fee0, 0xc43a73ff20, 0x7fad6035c8e0, 0xc435ad2330, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435ad2330, 0xa74a80, 0xc43f2e5fc0, 0x7fad6035c8e0, 0xc435ad2330, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc435ad2330, 0xa74a80, 0xc43f2e5fc0, 0xc43a73fee0, 0xc43a73ff40, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xffffffff, 0x0, 0x19, 0xc4422717e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xc43a73fee0, 0xc43a73ff60, 0xc442271878, 0x558012, 0x588a58c7, 0x371d1501)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xc442271900, 0x0, 0x0, 0x57)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xc43a73fee0, 0xc43a752ae0, 0xa91b40, 0xc43a752b20, 0xc43a73fee0, 0xc43a752b20)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xc420246100, 0xc4373a70b0, 0xc442271a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc435b26210, 0xa74a80, 0xc43f2e5fc0, 0xc43a73fee0, 0xc43a73ff20, 0x3c47, 0xc420239b18, 0x0, 0xc420239b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc435b26210, 0xc4325b32c0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc100, 0xc420331380, 0x10265c0, 0xc435b26270, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc100, 0xc435b26270)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434cdc100)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 12731 [semacquire]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43517bbf0, 0xc43517bc20, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43517bbf0, 0xc446896800, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43517bbf0, 0xc44634a520, 0x0, 0xc428ac99b0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43517bbf0, 0xa74a80, 0xc44634a520, 0x7fad6035c8e0, 0xc43517bbf0, 0x1038500, 0xc43517bbc0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43517bbc0, 0xa74a80, 0xc44634a520, 0xc4468966e0, 0xc446896700, 0x7fad6035c8e0, 0xc43517bbc0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43517bbc0, 0xa74a80, 0xc44634a520, 0x7fad6035c8e0, 0xc43517bbc0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43517bbc0, 0xa74a80, 0xc44634a520, 0xc4468966e0, 0xc446896720, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xffffffff, 0x0, 0x19, 0xc43530f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xc4468966e0, 0xc446896740, 0xc43530f878, 0x558012, 0x588a5a4a, 0x11c7020f)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xed01c514a, 0x0, 0x0, 0x9ee520)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xc4468966e0, 0xc446896760, 0xa91b40, 0xc4468967a0, 0xc4468966e0, 0xc4468967a0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xc420246100, 0xc42ec33740, 0xc43530fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43c8273b0, 0xa74a80, 0xc44634a520, 0xc4468966e0, 0xc446896700, 0x788a, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43c8273b0, 0xc440cd2720, 0xc420179d60, 0x0, 0x0, 0x10, 0xc437c4ef58)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7200, 0xc420331380, 0x10265c0, 0xc43c8273e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7200, 0xc43c8273e0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4396a7200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 11292 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc429dd08a0, 0xc429dd0900, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc429dd08a0, 0xc43c0a46e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc429dd08a0, 0xc43f3a82a0, 0x0, 0xc432fa2c30, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc429dd08a0, 0xa74a80, 0xc43f3a82a0, 0x7fad6035c8e0, 0xc429dd08a0, 0x1038500, 0xc429dd07e0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc429dd07e0, 0xa74a80, 0xc43f3a82a0, 0xc43c0a45c0, 0xc43c0a45e0, 0x7fad6035c8e0, 0xc429dd07e0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc429dd07e0, 0xa74a80, 0xc43f3a82a0, 0x7fad6035c8e0, 0xc429dd07e0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc429dd07e0, 0xa74a80, 0xc43f3a82a0, 0xc43c0a45c0, 0xc43c0a4600, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xffffffff, 0x0, 0x19, 0xc437c7f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xc43c0a45c0, 0xc43c0a4620, 0xc437c7f878, 0x558012, 0x588a5978, 0x12ac6d23)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xed01c5078, 0x0, 0x0, 0xca46b)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xc43c0a45c0, 0xc43c0a4640, 0xa91b40, 0xc43c0a4660, 0xc43c0a45c0, 0xc43c0a4660)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xc420246100, 0xc4331ed550, 0xc437c7fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a29aab0, 0xa74a80, 0xc43f3a82a0, 0xc43c0a45c0, 0xc43c0a45e0, 0x7876, 0xc420e1fb18, 0x0, 0xc420e1fb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a29aab0, 0xc442bd4300, 0xc420179d60, 0x0, 0x0, 0x65646f6e040a230a, 0x3237312d70691b12)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa400, 0xc420331380, 0x10265c0, 0xc42a2586c0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa400, 0xc42a2586c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fa400)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 10457 [chan receive, 6 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc448f6d860, 0xc42c051590)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc448f6d860, 0xc42c051590)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc448f6d860, 0xc42c051590)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc44873be20, 0xc42c051590)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc44873be20, 0xc42c051590)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc44873be20, 0xc42c051590)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc44873be00, 0xc42c0513b0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc44873be00, 0xc42c0513b0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc44873bde0, 0xc42c0513b0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc44873bde0, 0xc42c0513b0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc4493e71e0, 0xc42c0513b0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc4493e71e0, 0xc42c0512c0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc4493e71e0, 0xc42c0512c0)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc430c1be80, 0x1038440, 0xc441cbf1c0)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 11419 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc447de05d0, 0xc447de0600, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc447de05d0, 0xc442adc980, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc447de05d0, 0xc43e0c0600, 0x0, 0xc431f190b8, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc447de05d0, 0xa74a80, 0xc43e0c0600, 0x7fad6035c8e0, 0xc447de05d0, 0x1038500, 0xc447de05a0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc447de05a0, 0xa74a80, 0xc43e0c0600, 0xc442adc860, 0xc442adc880, 0x7fad6035c8e0, 0xc447de05a0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc447de05a0, 0xa74a80, 0xc43e0c0600, 0x7fad6035c8e0, 0xc447de05a0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc447de05a0, 0xa74a80, 0xc43e0c0600, 0xc442adc860, 0xc442adc8a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xffffffff, 0x0, 0x19, 0xc4361897e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xc442adc860, 0xc442adc8c0, 0xc436189878, 0x558012, 0x588a597c, 0x47ef91)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xed01c507c, 0x0, 0x0, 0x99565)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xc442adc860, 0xc442adc8e0, 0xa91b40, 0xc442adc900, 0xc442adc860, 0xc442adc900)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xc420246100, 0xc431f9b760, 0xc436189a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc440bce8a0, 0xa74a80, 0xc43e0c0600, 0xc442adc860, 0xc442adc880, 0x4bc6, 0xc420318b18, 0x0, 0xc420318b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc440bce8a0, 0xc439a5dce0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41ed5c013)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438723200, 0xc420331380, 0x10265c0, 0xc440bce8d0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438723200, 0xc440bce8d0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc438723200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 12080 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc442d01dd0, 0xc442d01e00, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc442d01dd0, 0xc441f79ae0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc442d01dd0, 0xc442f320e0, 0x0, 0xc428dc6280, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc442d01dd0, 0xa74a80, 0xc442f320e0, 0x7fad6035c8e0, 0xc442d01dd0, 0x1038500, 0xc442d01da0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc442d01da0, 0xa74a80, 0xc442f320e0, 0xc441f799e0, 0xc441f79a00, 0x7fad6035c8e0, 0xc442d01da0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d01da0, 0xa74a80, 0xc442f320e0, 0x7fad6035c8e0, 0xc442d01da0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc442d01da0, 0xa74a80, 0xc442f320e0, 0xc441f799e0, 0xc441f79a20, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xffffffff, 0x0, 0x19, 0xc435e097e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xc441f799e0, 0xc441f79a40, 0xc435e09878, 0x558012, 0x588a59f3, 0x37b6149d)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xed01c50f3, 0x0, 0x0, 0x73c98)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xc441f799e0, 0xc441f79a60, 0xa91b40, 0xc441f79a80, 0xc441f799e0, 0xc441f79a80)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xc420246100, 0xc4295c22c0, 0xc435e09a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442d001b0, 0xa74a80, 0xc442f320e0, 0xc441f799e0, 0xc441f79a00, 0x472f, 0xc435e50b18, 0x0, 0xc435e50b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442d001b0, 0xc4445a80c0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41fa28d4b)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438064000, 0xc420331380, 0x10265c0, 0xc442d001e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438064000, 0xc442d001e0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc438064000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 12391 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc439e9b2c0, 0xc439e9b2f0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc439e9b2c0, 0xc4414e1860, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc439e9b2c0, 0xc442b7e000, 0x0, 0xc439e71070, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc439e9b2c0, 0xa74a80, 0xc442b7e000, 0x7fad6035c8e0, 0xc439e9b2c0, 0x1038500, 0xc439e9b290)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc439e9b290, 0xa74a80, 0xc442b7e000, 0xc4414e1760, 0xc4414e1780, 0x7fad6035c8e0, 0xc439e9b290, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc439e9b290, 0xa74a80, 0xc442b7e000, 0x7fad6035c8e0, 0xc439e9b290, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc439e9b290, 0xa74a80, 0xc442b7e000, 0xc4414e1760, 0xc4414e17a0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xffffffff, 0x0, 0x19, 0xc4353677e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xc4414e1760, 0xc4414e17c0, 0xc435367878, 0x558012, 0x588a5a0e, 0x11594457)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xc435367900, 0x0, 0x0, 0xbf)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xc4414e1760, 0xc4414e17e0, 0xa91b40, 0xc4414e1800, 0xc4414e1760, 0xc4414e1800)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xc420246100, 0xc43a105c40, 0xc435367a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43be19320, 0xa74a80, 0xc442b7e000, 0xc4414e1760, 0xc4414e1780, 0x7c58, 0xc420049b18, 0x0, 0xc420049b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43be19320, 0xc4377bdbc0, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1c00, 0xc420331380, 0x10265c0, 0xc43be19350, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1c00, 0xc43be19350)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc434aa1c00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8517 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42e940180, 0xc42e9401b0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42e940180, 0xc4405a54e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42e940180, 0xc433c72ec0, 0x0, 0xc42e9daaf0, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42e940180, 0xa74a80, 0xc433c72ec0, 0x7fad6035c8e0, 0xc42e940180, 0x1038500, 0xc42e940150)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42e940150, 0xa74a80, 0xc433c72ec0, 0xc4405a53e0, 0xc4405a5400, 0x7fad6035c8e0, 0xc42e940150, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42e940150, 0xa74a80, 0xc433c72ec0, 0x7fad6035c8e0, 0xc42e940150, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42e940150, 0xa74a80, 0xc433c72ec0, 0xc4405a53e0, 0xc4405a5420, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xffffffff, 0x0, 0x19, 0xc448c2f7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xc4405a53e0, 0xc4405a5440, 0xc448c2f878, 0x558012, 0x588a57b9, 0x3b1ba526)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xed01c4eb9, 0x0, 0x0, 0xd01ee)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xc4405a53e0, 0xc4405a5460, 0xa91b40, 0xc4405a5480, 0xc4405a53e0, 0xc4405a5480)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xc420246100, 0xc42443bbb0, 0xc448c2fa70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42210a6c0, 0xa74a80, 0xc433c72ec0, 0xc4405a53e0, 0xc4405a5400, 0x7a83, 0xc420e18b18, 0x0, 0xc420e18b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42210a6c0, 0xc42693db00, 0xc420179d60, 0x0, 0x0, 0x4d1279726f746973, 0x6f642e7865646e69)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422b4f000, 0xc420331380, 0x10265c0, 0xc42210a6f0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422b4f000, 0xc42210a6f0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc422b4f000)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 12198 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4231ee37c)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc4231ee370)
	/usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc4231ee370, 0xc4445a86c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e

goroutine 12197 [chan send, 2 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc43cb882d0, 0xc4445a8660, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 8111 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc440b30930, 0xc440b30960, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc440b30930, 0xc438ec47e0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc440b30930, 0xc43900d1e0, 0x0, 0xc440198100, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc440b30930, 0xa74a80, 0xc43900d1e0, 0x7fad6035c8e0, 0xc440b30930, 0x1038500, 0xc440b30900)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc440b30900, 0xa74a80, 0xc43900d1e0, 0xc438ec46a0, 0xc438ec46c0, 0x7fad6035c8e0, 0xc440b30900, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440b30900, 0xa74a80, 0xc43900d1e0, 0x7fad6035c8e0, 0xc440b30900, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc440b30900, 0xa74a80, 0xc43900d1e0, 0xc438ec46a0, 0xc438ec4700, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xffffffff, 0x0, 0x19, 0xc4200597e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xc438ec46a0, 0xc438ec4720, 0xc420059878, 0x558012, 0x588a577a, 0xc811a11)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xed01c4e7a, 0x0, 0x0, 0x952a7)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xc438ec46a0, 0xc438ec4740, 0xa91b40, 0xc438ec4760, 0xc438ec46a0, 0xc438ec4760)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xc420246100, 0xc441a503a0, 0xc420059a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43414e5a0, 0xa74a80, 0xc43900d1e0, 0xc438ec46a0, 0xc438ec46c0, 0x3e9c, 0xc420e1eb18, 0x0, 0xc420e1eb08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43414e5a0, 0xc43b0b85a0, 0xc420179d60, 0x0, 0x0, 0xc4462a7fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aae00, 0xc420331380, 0x10265c0, 0xc43414e5d0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aae00, 0xc43414e5d0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc4267aae00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 11428 [chan send, 4 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc441ff1b30, 0xc43c3da780, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 5347 [semacquire, 13 minutes]:
sync.runtime_Semacquire(0xc4204895c4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4204895c0)
	/usr/local/go/src/sync/mutex.go:85 +0xd0
github.com/weaveworks/cortex/ingester.(*fingerprintLocker).Lock(0xc420235780, 0x475bf267ffed65dd)
	/go/src/github.com/weaveworks/cortex/ingester/locker.go:49 +0x6e
github.com/weaveworks/cortex/ingester.(*userState).unlockedGet(0xc42031cec0, 0xc44499fbf0, 0xc420162418, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:156 +0x50
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc44499f740, 0xc44499fbf0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:124 +0x260
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc44499f740, 0xc42b418560, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc44499f740, 0xc42b4a0820, 0x0, 0xc4446e8c10, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc44499f740, 0xa74a80, 0xc42b4a0820, 0x7fad6035c8e0, 0xc44499f740, 0x1038500, 0xc44499f710)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc44499f710, 0xa74a80, 0xc42b4a0820, 0xc42b435d60, 0xc42b435d80, 0x7fad6035c8e0, 0xc44499f710, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc44499f710, 0xa74a80, 0xc42b4a0820, 0x7fad6035c8e0, 0xc44499f710, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc44499f710, 0xa74a80, 0xc42b4a0820, 0xc42b435d60, 0xc42b435e00, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0xffffffff, 0x0, 0x19, 0xc4344eb7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0xc42b435d60, 0xc42b435e20, 0xc4344eb878, 0x558012, 0x588a575f, 0x36e7dde4)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0x411715, 0x0, 0x0, 0x200)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0xc42b435d60, 0xc42b435ea0, 0xa91b40, 0xc42b435ec0, 0xc42b435d60, 0xc42b435ec0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0xc420246100, 0xc444aea970, 0xc4344eba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc444ae7500, 0xa74a80, 0xc42b4a0820, 0xc42b435d60, 0xc42b435d80, 0xad5a, 0xc420313b18, 0x0, 0xc420313b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc444ae7500, 0xc448c5d800, 0xc420179d60, 0x0, 0x0, 0xc42ad71fa8, 0x4b2ba1)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbf400, 0xc420331380, 0x10265c0, 0xc444ae7530, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbf400, 0xc444ae7530)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc420cbf400)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8127 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc43d97727c)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc43d977270)
	/usr/local/go/src/sync/waitgroup.go:131 +0x97
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1(0xc43d977270, 0xc43b0b95c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:427 +0x2d
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:429 +0x27e

goroutine 8567 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4468dad20, 0xc4468dad50, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4468dad20, 0xc43b31a0a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4468dad20, 0xc43fb024e0, 0x0, 0xc42de30c50, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4468dad20, 0xa74a80, 0xc43fb024e0, 0x7fad6035c8e0, 0xc4468dad20, 0x1038500, 0xc4468dacf0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4468dacf0, 0xa74a80, 0xc43fb024e0, 0xc44272f620, 0xc44272f640, 0x7fad6035c8e0, 0xc4468dacf0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4468dacf0, 0xa74a80, 0xc43fb024e0, 0x7fad6035c8e0, 0xc4468dacf0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4468dacf0, 0xa74a80, 0xc43fb024e0, 0xc44272f620, 0xc44272f660, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xffffffff, 0x0, 0x19, 0xc439e2b7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xc44272f620, 0xc43b31a000, 0xc439e2b878, 0x558012, 0x588a59b8, 0x38bc914)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xc439e2b900, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xc44272f620, 0xc43b31a020, 0xa91b40, 0xc43b31a040, 0xc44272f620, 0xc43b31a040)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xc420246100, 0xc42ec40bd0, 0xc439e2ba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc440461680, 0xa74a80, 0xc43fb024e0, 0xc44272f620, 0xc44272f640, 0x4638, 0xc420314b18, 0x0, 0xc420314b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc440461680, 0xc44847ade0, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f600, 0xc420331380, 0x10265c0, 0xc4404616b0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f600, 0xc4404616b0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421f31b90, 0xc4202460c0, 0x103ac20, 0xc420246600, 0xc433f6f600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8291 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc42a8fd2f0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc42a8fd2f0, 0x159dc2dc318, 0x159dc659bf8, 0xc42c2482f0, 0x1, 0x1, 0xc42a8fd200, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:272 +0x7e
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc42a8fd2f0, 0xc42a8fd230, 0x0, 0xc43c598a50, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/cortex._Ingester_Query_Handler.func1(0x7fad6035c8e0, 0xc42a8fd2f0, 0xa934c0, 0xc42a8fd230, 0x7fad6035c8e0, 0xc42a8fd2f0, 0x1038500, 0xc42a8fd2c0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:439 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42a8fd2c0, 0xa934c0, 0xc42a8fd230, 0xc438220aa0, 0xc438220ac0, 0x7fad6035c8e0, 0xc42a8fd2c0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fd2c0, 0xa934c0, 0xc42a8fd230, 0x7fad6035c8e0, 0xc42a8fd2c0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42a8fd2c0, 0xa934c0, 0xc42a8fd230, 0xc438220aa0, 0xc438220ae0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xffffffff, 0x0, 0x19, 0xc4457777e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xc438220aa0, 0xc438220b20, 0xc445777878, 0x558012, 0x588a5791, 0x11ffde7e)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xed01c4e91, 0x0, 0x0, 0x93f5)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xc438220aa0, 0xc438220b40, 0xa91b40, 0xc438220b60, 0xc438220aa0, 0xc438220b60)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xc420246100, 0xc43c598a10, 0xc445777a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42a8fd110, 0xa934c0, 0xc42a8fd230, 0xc438220aa0, 0xc438220ac0, 0x41, 0xc420313b18, 0x0, 0xc420313b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Query_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42a8fd110, 0xc442ba8b40, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:441 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc4385fac00, 0xc420331380, 0x10265d8, 0xc42a8fd1d0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc4385fac00, 0xc42a8fd1d0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc43c1fd850, 0xc4202460c0, 0x103ac20, 0xc4271a6840, 0xc4385fac00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 11988 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc426bbdef0, 0xc426bbdf20, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc426bbdef0, 0xc43ee83220, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc426bbdef0, 0xc43ee82060, 0x0, 0xc4309ddd70, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc426bbdef0, 0xa74a80, 0xc43ee82060, 0x7fad6035c8e0, 0xc426bbdef0, 0x1038500, 0xc426bbdec0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc426bbdec0, 0xa74a80, 0xc43ee82060, 0xc43ee83120, 0xc43ee83140, 0x7fad6035c8e0, 0xc426bbdec0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc426bbdec0, 0xa74a80, 0xc43ee82060, 0x7fad6035c8e0, 0xc426bbdec0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc426bbdec0, 0xa74a80, 0xc43ee82060, 0xc43ee83120, 0xc43ee83160, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xffffffff, 0x0, 0x19, 0xc4377e17e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xc43ee83120, 0xc43ee83180, 0xc4377e1878, 0x558012, 0x588a59d5, 0x382e500c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xed01c50d5, 0x0, 0x0, 0x86ac6)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xc43ee83120, 0xc43ee831a0, 0xa91b40, 0xc43ee831c0, 0xc43ee83120, 0xc43ee831c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xc420246100, 0xc431261350, 0xc4377e1a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc42d3a5ce0, 0xa74a80, 0xc43ee82060, 0xc43ee83120, 0xc43ee83140, 0xd89, 0xc440d67b18, 0x0, 0xc440d67b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc42d3a5ce0, 0xc43d51a240, 0xc420179d60, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2600, 0xc420331380, 0x10265c0, 0xc42d3a5e00, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2600, 0xc42d3a5e00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4394f2600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8126 [chan send, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc441ff04b0, 0xc43b0b9560, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 8152 [chan send, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0xc42c0502d0, 0xc42c2128a0, 0x44a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x151
created by github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x16a

goroutine 8864 [semacquire, 10 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc4487ec240, 0xc4487ec270, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc4487ec240, 0xc44a3d32a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc4487ec240, 0xc44a514ca0, 0x0, 0xc4402dc670, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc4487ec240, 0xa74a80, 0xc44a514ca0, 0x7fad6035c8e0, 0xc4487ec240, 0x1038500, 0xc4487ec210)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc4487ec210, 0xa74a80, 0xc44a514ca0, 0xc44a3d31a0, 0xc44a3d31c0, 0x7fad6035c8e0, 0xc4487ec210, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc4487ec210, 0xa74a80, 0xc44a514ca0, 0x7fad6035c8e0, 0xc4487ec210, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc4487ec210, 0xa74a80, 0xc44a514ca0, 0xc44a3d31a0, 0xc44a3d31e0, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xffffffff, 0x0, 0x19, 0xc444c057e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xc44a3d31a0, 0xc44a3d3200, 0xc444c05878, 0x558012, 0x588a57f6, 0x42dfb09)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xed01c4ef6, 0x0, 0x0, 0xd2792)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xc44a3d31a0, 0xc44a3d3220, 0xa91b40, 0xc44a3d3240, 0xc44a3d31a0, 0xc44a3d3240)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xc420246100, 0xc449d20670, 0xc444c05a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc442487200, 0xa74a80, 0xc44a514ca0, 0xc44a3d31a0, 0xc44a3d31c0, 0x4d2c, 0xc420313b18, 0x0, 0xc420313b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc442487200, 0xc428978de0, 0xc420179d60, 0x0, 0x0, 0x50, 0xc41f1a9759)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87f00, 0xc420331380, 0x10265c0, 0xc442487230, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87f00, 0xc442487230)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc439b87f00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8338 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc42d1ab800, 0xc42d1ab830, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1ab800, 0xc437d58ae0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc42d1ab800, 0xc437da1440, 0x0, 0xc42b9ddd18, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc42d1ab800, 0xa74a80, 0xc437da1440, 0x7fad6035c8e0, 0xc42d1ab800, 0x1038500, 0xc42d1ab7d0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc42d1ab7d0, 0xa74a80, 0xc437da1440, 0xc437d589e0, 0xc437d58a00, 0x7fad6035c8e0, 0xc42d1ab7d0, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc42d1ab7d0, 0xa74a80, 0xc437da1440, 0x7fad6035c8e0, 0xc42d1ab7d0, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc42d1ab7d0, 0xa74a80, 0xc437da1440, 0xc437d589e0, 0xc437d58a20, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43a5d7ec0, 0xa74a80, 0xc437da1440, 0xffffffff, 0x0, 0x19, 0xc43b4f37e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43a5d7ec0, 0xa74a80, 0xc437da1440, 0xc437d589e0, 0xc437d58a40, 0xc43b4f3878, 0x558012, 0x588a5798, 0xdab9c82)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43a5d7ec0, 0xa74a80, 0xc437da1440, 0xed01c4e98, 0x0, 0x0, 0x16375a)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43a5d7ec0, 0xa74a80, 0xc437da1440, 0xc437d589e0, 0xc437d58a60, 0xa91b40, 0xc437d58a80, 0xc437d589e0, 0xc437d58a80)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43a5d7ec0, 0xa74a80, 0xc437da1440, 0xc420246100, 0xc42c8d4750, 0xc43b4f3a70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43a5d7ec0, 0xa74a80, 0xc437da1440, 0xc437d589e0, 0xc437d58a00, 0x8c54, 0xc420e18b18, 0x0, 0xc420e18b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43a5d7ec0, 0xc44363f860, 0xc420179d60, 0x0, 0x0, 0x30, 0x0)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fb600, 0xc420331380, 0x10265c0, 0xc43a5d7ef0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fb600, 0xc43a5d7ef0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4385fb600)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8125 [chan receive, 12 minutes]:
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc420064c80, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/registry.go:455 +0x53b
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.UninstrumentedHandler.func1(0x1036f00, 0xc42276b9f0, 0xc441ff04b0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:75 +0x47
net/http.HandlerFunc.ServeHTTP(0xb47360, 0x1036f00, 0xc42276b9f0, 0xc441ff04b0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(Handler).ServeHTTP-fm(0x1036f00, 0xc42276b9f0, 0xc441ff04b0)
	/usr/local/go/src/net/http/h2_bundle.go:4084 +0x4d
github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1037400, 0xc438a113c0, 0xc441ff04b0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2ab
net/http.HandlerFunc.ServeHTTP(0xc420171590, 0x1037400, 0xc438a113c0, 0xc441ff04b0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc420171270, 0x1037400, 0xc438a113c0, 0xc441ff04b0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/gorilla/mux/mux.go:109 +0x10d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Instrument.Wrap.func1(0x1037400, 0xc438a113a0, 0xc441ff02d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/instrument.go:43 +0x1a2
net/http.HandlerFunc.ServeHTTP(0xc420331590, 0x1037400, 0xc438a113a0, 0xc441ff02d0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.Log.Wrap.func1(0x1036e40, 0xc438a11360, 0xc441ff02d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/logging.go:35 +0x15c
net/http.HandlerFunc.ServeHTTP(0xc42033e3a0, 0x1036e40, 0xc438a11360, 0xc441ff02d0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func1(0x1037780, 0xc43d0740d0, 0xc441ff02d0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:37 +0x449
net/http.HandlerFunc.ServeHTTP(0xc4203315c0, 0x1037780, 0xc43d0740d0, 0xc441ff01e0)
	/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc42007ae80, 0x1037780, 0xc43d0740d0, 0xc441ff01e0)
	/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc422c7e000, 0x1038440, 0xc42d024a00)
	/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2293 +0x44d

goroutine 8124 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreateSeries(0xc4201e3078, 0x7fad6035c8e0, 0xc43d975260, 0xc43d975290, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:121 +0xad
github.com/weaveworks/cortex/ingester.(*Ingester).append(0xc4201e3040, 0x7fad6035c8e0, 0xc43d975260, 0xc438a10460, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:231 +0x208
github.com/weaveworks/cortex/ingester.(*Ingester).Push(0xc4201e3040, 0x7fad6035c8e0, 0xc43d975260, 0xc438e8ce20, 0x0, 0xc43d977120, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:205 +0x79
github.com/weaveworks/cortex._Ingester_Push_Handler.func1(0x7fad6035c8e0, 0xc43d975260, 0xa74a80, 0xc438e8ce20, 0x7fad6035c8e0, 0xc43d975260, 0x1038500, 0xc43d975230)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:421 +0xa2
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerUserHeaderInterceptor(0x7fad6035c8e0, 0xc43d975230, 0xa74a80, 0xc438e8ce20, 0xc438a10320, 0xc438a10340, 0x7fad6035c8e0, 0xc43d975230, 0xa3e980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:57 +0x15a
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d975230, 0xa74a80, 0xc438e8ce20, 0x7fad6035c8e0, 0xc43d975230, 0x103b980, 0x1074250)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x7fad6035c8e0, 0xc43d975230, 0xa74a80, 0xc438e8ce20, 0xc438a10320, 0xc438a10360, 0x0, 0x0, 0x102f900, 0xc420162ea0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:56 +0x39d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d298de0, 0xa74a80, 0xc438e8ce20, 0xffffffff, 0x0, 0x19, 0xc4260fb7e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerInstrumentInterceptor.func1(0x7fad6035c8e0, 0xc43d298de0, 0xa74a80, 0xc438e8ce20, 0xc438a10320, 0xc438a103a0, 0xc4260fb878, 0x558012, 0x588a577e, 0x428c614)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:15 +0x9e
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d298de0, 0xa74a80, 0xc438e8ce20, 0xed01c4e7e, 0x0, 0x0, 0x20a5b0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware.ServerLoggingInterceptor.func1(0x7fad6035c8e0, 0xc43d298de0, 0xa74a80, 0xc438e8ce20, 0xc438a10320, 0xc438a103c0, 0xa91b40, 0xc438a103e0, 0xc438a10320, 0xc438a103e0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:17 +0x9d
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x7fad6035c8e0, 0xc43d298de0, 0xa74a80, 0xc438e8ce20, 0xc420246100, 0xc43a078110, 0xc4260fba70, 0x410558)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:21 +0x66
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryServer.func1(0x7fad6035c8e0, 0xc43d298de0, 0xa74a80, 0xc438e8ce20, 0xc438a10320, 0xc438a10340, 0x86ee, 0xc420e18b18, 0x0, 0xc420e18b08)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:28 +0x118
github.com/weaveworks/cortex._Ingester_Push_Handler(0xad6580, 0xc4201e3040, 0x7fad6035c8e0, 0xc43d298de0, 0xc43b0b88a0, 0xc420179d60, 0x0, 0x0, 0x7fad625dc960, 0xc41f37d9fd)
	/go/src/github.com/weaveworks/cortex/cortex.pb.go:423 +0x166
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4275e7200, 0xc420331380, 0x10265c0, 0xc43d298e10, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:708 +0xbd8
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4275e7200, 0xc43d298e10)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:892 +0x6a4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201622c0, 0xc4202460c0, 0x103ac20, 0xc42000c300, 0xc4275e7200)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:468 +0xab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:469 +0xa3

goroutine 8239 [semacquire, 12 minutes]:
sync.runtime_Semacquire(0xc4201e3084)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*RWMutex).RLock(0xc4201e3078)
	/usr/local/go/src/sync/rwmutex.go:43 +0x5d
github.com/weaveworks/cortex/ingester.(*userStates).getOrCreate(0xc4201e3078, 0x7fad6035c8e0, 0xc43c59b320, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/user_state.go:96 +0x8f
github.com/weaveworks/cortex/ingester.(*Ingester).query(0xc4201e3040, 0x7fad6035c8e0, 0xc43c59b320, 0x159dc2e2c6f, 0x159dc66054f, 0xc42962f5d0, 0x1, 0x1, 0xc43c59b300, 0x1039120, ...)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:272 +0x7e
github.com/weaveworks/cortex/ingester.(*Ingester).Query(0xc4201e3040, 0x7fad6035c8e0, 0xc43c59b320, 0xc43c59b260, 0x0, 0xc43d794240, 0xa01e00)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:261 +0x9f
github.com/weaveworks/c