untitled.txt

=== received SIGQUIT ===
*** goroutine dump...
goroutine 1 [running]:
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/signals.SignalHandlerLoop(0x7fd2c8475028, 0xc4200106e0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/signals/signals.go:37 +0x35c
github.com/weaveworks/cortex/server.(*Server).Run(0xc4202cc1e0)
	/go/src/github.com/weaveworks/cortex/server/server.go:110 +0x474
main.main()
	/go/src/github.com/weaveworks/cortex/cmd/ingester/main.go:52 +0x473

goroutine 17 [syscall, 6829 minutes, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 21 [syscall]:
os/signal.signal_recv(0x1036040)
	/usr/local/go/src/runtime/sigqueue.go:116 +0x157
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 23 [select]:
net/http.(*persistConn).roundTrip(0xc4e0d93c00, 0xc45dece860, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc42028c000, 0xc525eec000, 0xc42028c000, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc525eec000, 0x1030bc0, 0xc42028c000, 0x0, 0x0, 0x0, 0x8, 0xc504345a40, 0xc51ab88970)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0xc4201b90e0, 0xc525eec000, 0x0, 0x0, 0x0, 0xc51ab88970, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0xc4201b90e0, 0xc525eec000, 0xb479c8, 0x3, 0x5890a601, 0x27711870)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Do(0xc4201b90e0, 0xc525eec000, 0x1050960, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:184 +0x1ea
github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api.(*Client).doRequest(0xc42018a5a0, 0xc49b980af0, 0xc504345d30, 0xc45dece7e0, 0x16, 0xc49b980af0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api/api.go:475 +0x8e
github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api.(*KV).getInternal(0xc42017c190, 0xc4200fd850, 0xf, 0x0, 0xc49b980aa0, 0xc420180000, 0xc504345e00, 0x410558, 0x50)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api/kv.go:141 +0x233
github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api.(*KV).Get(0xc42017c190, 0xc4200fd850, 0xf, 0xc49b980aa0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/hashicorp/consul/api/kv.go:75 +0x8c
github.com/weaveworks/cortex/ring.(*consulClient).WatchKey(0xc4201773a0, 0xc4200fd850, 0xf, 0xc42018a600, 0xc4200fd840)
	/go/src/github.com/weaveworks/cortex/ring/consul_client.go:292 +0x117
github.com/weaveworks/cortex/ring.(*prefixedConsulClient).WatchKey(0xc4201773c0, 0xaf3a7c, 0x4, 0xc42018a600, 0xc4200fd840)
	/go/src/github.com/weaveworks/cortex/ring/consul_client.go:353 +0xa6
github.com/weaveworks/cortex/ring.(*Ring).loop(0xc42018a6c0)
	/go/src/github.com/weaveworks/cortex/ring/ring.go:119 +0xbe
created by github.com/weaveworks/cortex/ring.New
	/go/src/github.com/weaveworks/cortex/ring/ring.go:96 +0x38e

goroutine 24 [select]:
github.com/weaveworks/cortex/ring.(*IngesterRegistration).heartbeat(0xc4202424d0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ring/ingester_lifecycle.go:224 +0x3bc
github.com/weaveworks/cortex/ring.(*IngesterRegistration).loop(0xc4202424d0)
	/go/src/github.com/weaveworks/cortex/ring/ingester_lifecycle.go:153 +0xb9
created by github.com/weaveworks/cortex/ring.RegisterIngester
	/go/src/github.com/weaveworks/cortex/ring/ingester_lifecycle.go:123 +0x31f

goroutine 82 [select]:
github.com/weaveworks/cortex/chunk.(*MemcacheClient).updateLoop(0xc42017b3b0, 0xdf8475800, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/memcache_client.go:78 +0x1fe
created by github.com/weaveworks/cortex/chunk.NewMemcacheClient
	/go/src/github.com/weaveworks/cortex/chunk/memcache_client.go:63 +0x24a

goroutine 84 [sleep]:
time.Sleep(0x4256fcbd5)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc542b98e70, 0xc52c216ae0, 0x4, 0x4)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc542b98e70, 0xc4200177e8, 0x1, 0xc4f9ad8120, 0x4, 0x4, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc542b98e70, 0xc4f9ad8120, 0x4, 0x4, 0xc542b98f30, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc542b98e70, 0xf1d3b296ff1402a8, 0xc42c770030, 0xc47b9e3560, 0x4, 0x4, 0xc548135ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xf1d3b296ff1402a8, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 86 [sleep]:
time.Sleep(0xb2d88b954)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc43e679830, 0xc4ab768b40, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc43e679830, 0xc4200177e8, 0x1, 0xc50e34bcb0, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc43e679830, 0xc50e34bcb0, 0x2, 0x2, 0xc43e6798f0, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc43e679830, 0x99c3a5dd0879b274, 0xc429ddd680, 0xc4e9ea97b0, 0x2, 0x2, 0xc5371a9ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x99c3a5dd0879b274, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x2)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 2617 [IO wait]:
net.runtime_pollWait(0x7fd2c969fc50, 0x72, 0x9)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc421e77560, 0x72, 0xc4200458d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc421e77560, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc421e77500, 0xc421b0a000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420cacab8, 0xc421b0a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc4210190e0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc4210190e0, 0xc420244878, 0x9, 0x9, 0x10, 0xc525b0e3a0, 0x0)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc4210190e0, 0xc420244878, 0x9, 0x9, 0x9, 0xa984a0, 0x402801, 0xc525b0e3a0)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc4210190e0, 0xc420244878, 0x9, 0x9, 0x4000451e4201, 0xfc00000000000004, 0x5)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc420244878, 0x9, 0x9, 0x102f380, 0xc4210190e0, 0x0, 0xc400000000, 0x42a18c, 0xc420045be8)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420244840, 0x400001034e00, 0x400000801, 0x801000037ab, 0x37ab00000004)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4208a1ce0, 0xc525b0e3a0, 0xc525b0e3a0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420244900, 0xc4208a1e30, 0xb474b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202440c0, 0x103ac20, 0xc420244900)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202440c0, 0x103ae60, 0xc420cacab8, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202440c0, 0x103ae60, 0xc420cacab8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 85 [sleep]:
time.Sleep(0x7114f001)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc45802def0, 0xc4c8b8a1e0, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc45802def0, 0xc4200177e8, 0x1, 0xc51c80e480, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc45802def0, 0xc51c80e480, 0x2, 0x2, 0xc45802df50, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc45802def0, 0x3b1ba7202e47cbf1, 0xc42d089620, 0xc504dd1820, 0x2, 0x2, 0xc44ce3fee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x3b1ba7202e47cbf1, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x1)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 81 [select]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc42000c300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 87 [sleep]:
time.Sleep(0x2faf080)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc49f3c13b0, 0xc47aeabdd0, 0x4, 0x4)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc49f3c13b0, 0xc4200177e8, 0x1, 0xc4bc6c67e0, 0x4, 0x4, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc49f3c13b0, 0xc4bc6c67e0, 0x4, 0x4, 0xc49f3c1470, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc49f3c13b0, 0x2dbdaee81acfea1f, 0xc4267b8540, 0xc4942e2fc0, 0x4, 0x4, 0xc54ebf7ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x2dbdaee81acfea1f, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x3)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 88 [sleep]:
time.Sleep(0x69539c7)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc49e7599e0, 0xc4d98b6b70, 0x1, 0x1)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc49e7599e0, 0xc4200177e8, 0x1, 0xc4ae6ae910, 0x1, 0x1, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc49e7599e0, 0xc4ae6ae910, 0x1, 0x1, 0xc49e759a10, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc49e7599e0, 0x2d2e0d1933d1ee98, 0xc4903c7320, 0xc4b5032980, 0x1, 0x2, 0xc5446bfee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x2d2e0d1933d1ee98, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x4)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 89 [sleep]:
time.Sleep(0x6c6e9c489)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc511d167e0, 0xc44624e7e0, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc511d167e0, 0xc4200177e8, 0x1, 0xc42b238fc0, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc511d167e0, 0xc42b238fc0, 0x2, 0x2, 0xc511d16870, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc511d167e0, 0xf86d5c3c5d5c0781, 0xc49b1ab560, 0xc457604ae0, 0x2, 0x2, 0xc4ae539ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xf86d5c3c5d5c0781, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x5)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 90 [sleep]:
time.Sleep(0xba43b7400)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4b82f4d20, 0xc4c4bb4a20, 0x8, 0x8)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4b82f4d20, 0xc4200177e8, 0x1, 0xc540b7f440, 0x8, 0x8, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4b82f4d20, 0xc540b7f440, 0x8, 0x8, 0xc4b82f4ed0, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4b82f4d20, 0xd9c32e37d76af098, 0xc48b57de90, 0xc4b3eb4420, 0x8, 0xc, 0xc4390e5ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xd9c32e37d76af098, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x6)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 91 [sleep]:
time.Sleep(0xba43b7400)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4849a0510, 0xc48b7c5350, 0x7, 0x7)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4849a0510, 0xc42048ffad, 0x1, 0xc51fb81400, 0x7, 0x7, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4849a0510, 0xc51fb81400, 0x7, 0x7, 0xc4849a0720, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4849a0510, 0x7a8f575dcb5d34d1, 0xc4327f5980, 0xc44ecd2b40, 0x7, 0xc, 0xc53d499ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc42048ffad, 0x1, 0x7a8f575dcb5d34d1, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x7)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 92 [sleep]:
time.Sleep(0xba43b7400)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc445854120, 0xc485678300, 0x4, 0x4)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc445854120, 0xc4200177e8, 0x1, 0xc4408b3e60, 0x4, 0x4, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc445854120, 0xc4408b3e60, 0x4, 0x4, 0xc445854240, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc445854120, 0xb4f29d4a7e52eba2, 0xc42b44d8c0, 0xc4d3c26b40, 0x4, 0x4, 0xc5326a3ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xb4f29d4a7e52eba2, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x8)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 93 [sleep]:
time.Sleep(0x78aa1be8)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4b3295f80, 0xc4cac03050, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4b3295f80, 0xc4200177e8, 0x1, 0xc4dfbddc20, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4b3295f80, 0xc4dfbddc20, 0x2, 0x2, 0xc4c4f14000, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4b3295f80, 0x6b72bd0c568cd3af, 0xc47ece3ec0, 0xc469a2f430, 0x2, 0x2, 0xc52f7e5ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x6b72bd0c568cd3af, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x9)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 94 [sleep]:
time.Sleep(0x926273ca5)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc42f239dd0, 0xc42c3ba330, 0xc, 0xc)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc42f239dd0, 0xc4200177e8, 0x1, 0xc540ae0380, 0xc, 0xc, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc42f239dd0, 0xc540ae0380, 0xc, 0xc, 0xc42f1aa030, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc42f239dd0, 0xc7d83a4d95de1d00, 0xc47940b650, 0xc4b674b680, 0xc, 0x10, 0xc5063d7ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xc7d83a4d95de1d00, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0xa)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 95 [chan receive]:
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks(0xc42030a000, 0x7fd2c84ff6a0, 0xc4f45bb0e0, 0xc4200177e8, 0x1, 0xc42018f0a0, 0x3, 0x3, 0x1, 0x103b3e0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:359 +0x1f1
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4f45bb0e0, 0xc42018f0a0, 0x3, 0x3, 0xc4f45bb1a0, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:340 +0xb9
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4f45bb0e0, 0x7c2e738c3e927015, 0xc46f2b0150, 0xc490cf8740, 0x3, 0x4, 0xc542ae1ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x7c2e738c3e927015, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0xb)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 96 [sleep]:
time.Sleep(0x23ef64ba7)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc43183be90, 0xc431009530, 0xf, 0xf)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc43183be90, 0xc4200177e8, 0x1, 0xc52da53200, 0xf, 0xf, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc43183be90, 0xc52da53200, 0xf, 0xf, 0xc431818330, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc43183be90, 0x509914efba9d7abe, 0xc448c0dd10, 0xc448fba180, 0xf, 0x10, 0xc4efadbee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x509914efba9d7abe, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0xc)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 97 [sleep, 1 minutes]:
time.Sleep(0xba43b7400)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4579ba150, 0xc46e215b60, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4579ba150, 0xc4200177e8, 0x1, 0xc44d99db90, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4579ba150, 0xc44d99db90, 0x2, 0x2, 0xc4579ba1b0, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4579ba150, 0x201074bfaa0d084d, 0xc47bf24420, 0xc47b256ee0, 0x2, 0x2, 0xc50769dee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x201074bfaa0d084d, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0xd)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 98 [sleep]:
time.Sleep(0x1e90c4aa)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc49c4896e0, 0xc4b2eb06c0, 0x1, 0x1)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc49c4896e0, 0xc4200177e8, 0x1, 0xc4eca67220, 0x1, 0x1, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc49c4896e0, 0xc4eca67220, 0x1, 0x1, 0xc49c489710, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc49c4896e0, 0x3a316d7984c9c278, 0xc4c4bcdf20, 0xc4cb51fa80, 0x1, 0x2, 0xc4d656fee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x3a316d7984c9c278, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0xe)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 99 [sleep]:
time.Sleep(0xb23bcc06)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4c960ce10, 0xc4a8be0960, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4c960ce10, 0xc4200177e8, 0x1, 0xc4caafd710, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4c960ce10, 0xc4caafd710, 0x2, 0x2, 0xc4c960ce70, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4c960ce10, 0xf8eea24d8c225d53, 0xc42b582ea0, 0xc4fe2c4f30, 0x2, 0x2, 0xc51a6e1ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xf8eea24d8c225d53, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0xf)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 100 [chan receive]:
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks(0xc42030a000, 0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xc42048ffad, 0x1, 0xc47962fb00, 0x4, 0x4, 0x1, 0x103b3e0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:359 +0x1f1
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xc47962fb00, 0x4, 0x4, 0xc4a1ad0570, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:340 +0xb9
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4a1ad04b0, 0x6d708307bafc6940, 0xc48ccc22d0, 0xc441f00ec0, 0x4, 0x8, 0xc539561ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc42048ffad, 0x1, 0x6d708307bafc6940, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x10)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 101 [sleep]:
time.Sleep(0x73d1869)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4b0c398c0, 0xc4b91fa660, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4b0c398c0, 0xc4200177e8, 0x1, 0xc502dba6c0, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4b0c398c0, 0xc502dba6c0, 0x2, 0x2, 0xc4b0c39920, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4b0c398c0, 0x182d295c3c93c533, 0xc47bd505a0, 0xc462d10f50, 0x2, 0x2, 0xc4942abee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x182d295c3c93c533, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x11)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 102 [sleep]:
time.Sleep(0x4c8cb13)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc475003320, 0xc4a440e210, 0x3, 0x3)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc475003320, 0xc4200177e8, 0x1, 0xc472f54b60, 0x3, 0x3, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc475003320, 0xc472f54b60, 0x3, 0x3, 0xc4750033b0, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc475003320, 0x5d01e17f14c7d956, 0xc4423675c0, 0xc4f827b040, 0x3, 0x4, 0xc530993ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x5d01e17f14c7d956, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x12)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 103 [sleep]:
time.Sleep(0x46da69c2)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc431271620, 0xc51b8a6330, 0x5, 0x5)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc431271620, 0xc4200177e8, 0x1, 0xc547f78180, 0x5, 0x5, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc431271620, 0xc547f78180, 0x5, 0x5, 0xc431271770, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc431271620, 0xef822a5496a3a8cb, 0xc426e4c2d0, 0xc50dba8780, 0x5, 0x8, 0xc54ebfbee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xef822a5496a3a8cb, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x13)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 104 [sleep, 1 minutes]:
time.Sleep(0xba43b7400)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc45b336a20, 0xc4b003b680, 0x7, 0x7)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc45b336a20, 0xc4200177e8, 0x1, 0xc458be4e00, 0x7, 0x7, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc45b336a20, 0xc458be4e00, 0x7, 0x7, 0xc45b336ea0, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc45b336a20, 0x7fad0cc92b23235e, 0xc451b58390, 0xc4ca193300, 0x7, 0x8, 0xc55694fee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x7fad0cc92b23235e, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x14)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 105 [sleep]:
time.Sleep(0x93063993b)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4b5c71860, 0xc525bf5320, 0x3, 0x3)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4b5c71860, 0xc4200177e8, 0x1, 0xc472f55c00, 0x3, 0x3, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4b5c71860, 0xc472f55c00, 0x3, 0x3, 0xc4b5c718f0, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4b5c71860, 0xddb3c0ac186b044d, 0xc4241e25d0, 0xc447d01200, 0x3, 0x4, 0xc52ed4dee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xddb3c0ac186b044d, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x15)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 106 [sleep]:
time.Sleep(0xa7c255f)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4a7182750, 0xc4c2f2ed50, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4a7182750, 0xc42048ffad, 0x1, 0xc4a099ec60, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4a7182750, 0xc4a099ec60, 0x2, 0x2, 0xc4a71827b0, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4a7182750, 0xb130fb2778af857c, 0xc4bca64210, 0xc446cf3760, 0x2, 0x4, 0xc54bd47ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc42048ffad, 0x1, 0xb130fb2778af857c, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x16)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 107 [sleep]:
time.Sleep(0x1f7b71245)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc50782be90, 0xc4e746ea80, 0x3, 0x3)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc50782be90, 0xc4200177e8, 0x1, 0xc46fc31dc0, 0x3, 0x3, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc50782be90, 0xc46fc31dc0, 0x3, 0x3, 0xc50782bf20, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc50782be90, 0x8e43e0d337918425, 0xc442373dd0, 0xc4e712cf60, 0x3, 0x4, 0xc4fd065ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x8e43e0d337918425, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x17)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 108 [sleep]:
time.Sleep(0x69cf2f3)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc466145140, 0xc473f5c450, 0x1, 0x1)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc466145140, 0xc4200177e8, 0x1, 0xc46856d270, 0x1, 0x1, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc466145140, 0xc46856d270, 0x1, 0x1, 0xc466145170, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc466145140, 0x2e660548d9882c20, 0xc49271f9e0, 0xc4c8554380, 0x1, 0x2, 0xc555843ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x2e660548d9882c20, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x18)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 109 [sleep]:
time.Sleep(0xba43b7400)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc518aea810, 0xc538393a70, 0x7, 0x7)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc518aea810, 0xc4200177e8, 0x1, 0xc51f5e8c00, 0x7, 0x7, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc518aea810, 0xc51f5e8c00, 0x7, 0x7, 0xc518aea9f0, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc518aea810, 0xa2e993d5f02e83c1, 0xc42e67b470, 0xc469c51a00, 0x7, 0x8, 0xc52bb0dee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xa2e993d5f02e83c1, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x19)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 110 [sleep]:
time.Sleep(0x340199163)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4722feb10, 0xc45cf35800, 0x3, 0x3)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4722feb10, 0xc4200177e8, 0x1, 0xc5496c7960, 0x3, 0x3, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4722feb10, 0xc5496c7960, 0x3, 0x3, 0xc4722fec30, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4722feb10, 0x3edc42dacf7d7d44, 0xc48befa4e0, 0xc4dfd00c60, 0x3, 0x4, 0xc52a7c7ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x3edc42dacf7d7d44, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x1a)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 111 [sleep]:
time.Sleep(0x865fc89)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc49ec6b200, 0xc476870810, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc49ec6b200, 0xc4200177e8, 0x1, 0xc4832de1b0, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc49ec6b200, 0xc4832de1b0, 0x2, 0x2, 0xc49ec6b260, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc49ec6b200, 0x246a6c7b89e75ca9, 0xc4248d0030, 0xc531f0af20, 0x2, 0x2, 0xc52c379ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x246a6c7b89e75ca9, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x1b)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 112 [sleep]:
time.Sleep(0x3276ec046)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc554d01e60, 0xc52a2260f0, 0x5, 0x5)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc554d01e60, 0xc4200177e8, 0x1, 0xc508789500, 0x5, 0x5, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc554d01e60, 0xc508789500, 0x5, 0x5, 0xc554d01f50, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc554d01e60, 0x45baebc636b87898, 0xc43643ae70, 0xc4c7f77890, 0x5, 0x6, 0xc5446c3ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x45baebc636b87898, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x1c)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 113 [sleep]:
time.Sleep(0x1621a38e0)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4d8a3f4a0, 0xc4c73beb40, 0x1, 0x1)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4d8a3f4a0, 0xc4200177e8, 0x1, 0xc4e3c5c320, 0x1, 0x1, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4d8a3f4a0, 0xc4e3c5c320, 0x1, 0x1, 0xc4d8a3f4d0, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4d8a3f4a0, 0xea1641d8ec5c81eb, 0xc4afa2d8f0, 0xc45629a9b0, 0x1, 0x2, 0xc530be3ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xea1641d8ec5c81eb, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x1d)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 114 [sleep]:
time.Sleep(0xba43b7400)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc476d9e690, 0xc4e101c030, 0x4, 0x4)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc476d9e690, 0xc4200177e8, 0x1, 0xc4a0af2d80, 0x4, 0x4, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc476d9e690, 0xc4a0af2d80, 0x4, 0x4, 0xc476d9e750, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc476d9e690, 0xc9d7731ef764cfbe, 0xc4347a0c90, 0xc4d3d8eb40, 0x4, 0x6, 0xc5286e1ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xc9d7731ef764cfbe, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x1e)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 115 [sleep]:
time.Sleep(0xba43b7400)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc50e595140, 0xc51947cea0, 0x14, 0x14)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc50e595140, 0xc4200177e8, 0x1, 0xc4d9c88c00, 0x14, 0x14, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc50e595140, 0xc4d9c88c00, 0x14, 0x14, 0xc50e595530, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc50e595140, 0xa3254e62b311c5f7, 0xc43357df50, 0xc4d747f300, 0x14, 0x20, 0xc4d1eb1ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xa3254e62b311c5f7, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x1f)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 116 [sleep]:
time.Sleep(0x47ae65657)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc523a0a060, 0xc523a0af90, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc523a0a060, 0xc4200177e8, 0x1, 0xc52ab60fc0, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc523a0a060, 0xc52ab60fc0, 0x2, 0x2, 0xc523a0a0c0, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc523a0a060, 0xd77ae4cef4931392, 0xc46dd1d320, 0xc4862ef6c0, 0x2, 0x2, 0xc5446c3ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xd77ae4cef4931392, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x20)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 117 [sleep]:
time.Sleep(0xba43b7400)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc54875c870, 0xc4e824bfb0, 0xe, 0xe)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc54875c870, 0xc4200177e8, 0x1, 0xc5477fe400, 0xe, 0xe, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc54875c870, 0xc5477fe400, 0xe, 0xe, 0xc54875cb10, 0x2e)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc54875c870, 0x3caf615a621a79b, 0xc44dbd2120, 0xc45591b800, 0xe, 0x10, 0xc4390e1ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x3caf615a621a79b, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x21)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 118 [sleep]:
time.Sleep(0x7480c00e1)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc45a386c90, 0xc45f9e19e0, 0xa, 0xa)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc45a386c90, 0xc4200177e8, 0x1, 0xc52d695800, 0xa, 0xa, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc45a386c90, 0xc52d695800, 0xa, 0xa, 0xc45a386e70, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc45a386c90, 0xdf11db1a1474cace, 0xc4246af6b0, 0xc50a6cfc00, 0xa, 0x10, 0xc52832bee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xdf11db1a1474cace, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x22)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 119 [sleep]:
time.Sleep(0x521e19af6)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4ee7210e0, 0xc4c6ab0840, 0x4, 0x4)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4ee7210e0, 0xc4200177e8, 0x1, 0xc44f57bd40, 0x4, 0x4, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4ee7210e0, 0xc44f57bd40, 0x4, 0x4, 0xc4ee7211a0, 0x2e)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4ee7210e0, 0xd31b00d2e284cd9, 0xc4962e0990, 0xc43d8778c0, 0x4, 0x4, 0xc5424a1ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xd31b00d2e284cd9, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x23)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 120 [sleep]:
time.Sleep(0x8934cfd0)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4d08de180, 0xc4d08dff80, 0x1, 0x1)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4d08de180, 0xc4200177e8, 0x1, 0xc452cfd4a0, 0x1, 0x1, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4d08de180, 0xc452cfd4a0, 0x1, 0x1, 0xc4d08de1b0, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4d08de180, 0x4b628a5b43429db8, 0xc42045e690, 0xc43ac9af68, 0x1, 0x1, 0xc530997ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x4b628a5b43429db8, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x24)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 121 [sleep]:
time.Sleep(0x480048f1)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc49f3be540, 0xc4cf8d0cc0, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc49f3be540, 0xc4200177e8, 0x1, 0xc490598480, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc49f3be540, 0xc490598480, 0x2, 0x2, 0xc49f3be5d0, 0x2e)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc49f3be540, 0xacf2a38744f72e9, 0xc46bc15f50, 0xc504ddf5c0, 0x2, 0x2, 0xc53fce9ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xacf2a38744f72e9, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x25)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 122 [sleep]:
time.Sleep(0x6c341a8)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4bf5ac4b0, 0xc49def53e0, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4bf5ac4b0, 0xc4200177e8, 0x1, 0xc4d8a40510, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4bf5ac4b0, 0xc4d8a40510, 0x2, 0x2, 0xc4bf5ac510, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4bf5ac4b0, 0x9e5a0c34317823c0, 0xc467318c00, 0xc4f95bf270, 0x2, 0x2, 0xc542addee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x9e5a0c34317823c0, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x26)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 123 [sleep]:
time.Sleep(0x4b66b14)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4b0c38930, 0xc4b22d5800, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4b0c38930, 0xc4200177e8, 0x1, 0xc4d8a41a70, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4b0c38930, 0xc4d8a41a70, 0x2, 0x2, 0xc4b0c38990, 0x2e)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4b0c38930, 0x644012197ce4357, 0xc42695ca50, 0xc4446348f0, 0x2, 0x2, 0xc52f7e9ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x644012197ce4357, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x27)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 124 [sleep]:
time.Sleep(0x10a6a49bf)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc514852d20, 0xc524f33950, 0x3, 0x3)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc514852d20, 0xc4200177e8, 0x1, 0xc5488a9340, 0x3, 0x3, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc514852d20, 0xc5488a9340, 0x3, 0x3, 0xc514852db0, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc514852d20, 0xbcf56e6c9ea2f84c, 0xc4824fb2c0, 0xc4c1d78020, 0x3, 0x4, 0xc4d656bee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xbcf56e6c9ea2f84c, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x28)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 125 [sleep]:
time.Sleep(0xcfeac62)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc481a470b0, 0xc49c51e1e0, 0x1, 0x1)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc481a470b0, 0xc4200177e8, 0x1, 0xc47c9f29b0, 0x1, 0x1, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc481a470b0, 0xc47c9f29b0, 0x1, 0x1, 0xc481a470e0, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc481a470b0, 0x109881071e8c8761, 0xc49cf42ba0, 0xc4b1ee1410, 0x1, 0x2, 0xc51fdb5ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x109881071e8c8761, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x29)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 126 [sleep]:
time.Sleep(0x535fb04)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4983b6630, 0xc4650899e0, 0x1, 0x1)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4983b6630, 0xc4200177e8, 0x1, 0xc5121c9a90, 0x1, 0x1, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4983b6630, 0xc5121c9a90, 0x1, 0x1, 0xc4983b6690, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4983b6630, 0x26b71cd71403afb4, 0xc4b4516bd0, 0xc523f4f850, 0x1, 0x2, 0xc54249dee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x26b71cd71403afb4, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x2a)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 127 [sleep]:
time.Sleep(0x1a4fb685b)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4531a2b10, 0xc4eb766e10, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4531a2b10, 0xc4200177e8, 0x1, 0xc420347950, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4531a2b10, 0xc420347950, 0x2, 0x2, 0xc4531a2c00, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4531a2b10, 0xfb2e065233bf5f5b, 0xc446b22c60, 0xc4c9ba5530, 0x2, 0x2, 0xc53fce5ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xfb2e065233bf5f5b, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x2b)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 128 [sleep]:
time.Sleep(0x579d96ba)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc4cb57a720, 0xc4d9928ff0, 0x1, 0x1)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc4cb57a720, 0xc4200177e8, 0x1, 0xc5168ae0f0, 0x1, 0x1, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc4cb57a720, 0xc5168ae0f0, 0x1, 0x1, 0xc4cb57a750, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc4cb57a720, 0x7352dde43c030104, 0xc477beb350, 0xc4bbb08060, 0x1, 0x2, 0xc52c37dee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x7352dde43c030104, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x2c)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 129 [sleep]:
time.Sleep(0xf5cfd803)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc42d2bbec0, 0xc4da4740f0, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc42d2bbec0, 0xc4200177e8, 0x1, 0xc517c843f0, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc42d2bbec0, 0xc517c843f0, 0x2, 0x2, 0xc42d2bbf50, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc42d2bbec0, 0xbad0a8fff40d3e3b, 0xc4943dd290, 0xc4fcbd0370, 0x2, 0x2, 0xc4707a5ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xbad0a8fff40d3e3b, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x2d)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 130 [sleep]:
time.Sleep(0x5629124bd)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc52de959b0, 0xc50e196360, 0x1, 0x1)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:255 +0x464
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc52de959b0, 0xc4200177e8, 0x1, 0xc527cf2550, 0x1, 0x1, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc52de959b0, 0xc527cf2550, 0x1, 0x1, 0xc52de959e0, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc52de959b0, 0x44b906e036427812, 0xc49b45e6c0, 0xc4fbdb1c08, 0x1, 0x1, 0xc4d656fee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x44b906e036427812, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x2e)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 131 [sleep]:
time.Sleep(0x5d664a92c)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc45faca3f0, 0xc50d145e90, 0xb, 0xb)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc45faca3f0, 0xc4200177e8, 0x1, 0xc46db53880, 0xb, 0xb, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc45faca3f0, 0xc46db53880, 0xb, 0xb, 0xc45faca600, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc45faca3f0, 0x3aaeae21a9a29d5f, 0xc48f1abb30, 0xc52c2c3d80, 0xb, 0x10, 0xc54e969ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x3aaeae21a9a29d5f, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x2f)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 132 [sleep]:
time.Sleep(0x7877d062a)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc45f1ccd50, 0xc4d10cc960, 0x2, 0x2)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc45f1ccd50, 0xc4200177e8, 0x1, 0xc4baf72f30, 0x2, 0x2, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc45f1ccd50, 0xc4baf72f30, 0x2, 0x2, 0xc45f1ccde0, 0x30)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc45f1ccd50, 0xaceec29967443e1c, 0xc42ced5140, 0xc456a9e230, 0x2, 0x2, 0xc528f69ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0xaceec29967443e1c, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x30)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 133 [sleep]:
time.Sleep(0x878da85cc)
	/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/weaveworks/cortex/chunk.(*dynamoDBBackoffClient).batchWriteDynamo(0xc4201b4310, 0x7fd2c84ff6a0, 0xc444d1b9e0, 0xc53f229140, 0xc, 0xc)
	/go/src/github.com/weaveworks/cortex/chunk/dynamodb_client.go:246 +0x4cf
github.com/weaveworks/cortex/chunk.(*AWSStore).updateIndex(0xc42030a000, 0x7fd2c84ff6a0, 0xc444d1b9e0, 0xc4200177e8, 0x1, 0xc487c0fc00, 0xc, 0xc, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:399 +0xac
github.com/weaveworks/cortex/chunk.(*AWSStore).Put(0xc42030a000, 0x7fd2c84ff6a0, 0xc444d1b9e0, 0xc487c0fc00, 0xc, 0xc, 0xc444d1bd40, 0x2f)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:345 +0x12c
github.com/weaveworks/cortex/ingester.(*Ingester).flushChunks(0xc42030e000, 0x7fd2c84ff6a0, 0xc444d1b9e0, 0x53e6187edce1d0c1, 0xc424a4e150, 0xc4a1963680, 0xc, 0x10, 0xc4ecff1ee8, 0x1034400)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:544 +0x39e
github.com/weaveworks/cortex/ingester.(*Ingester).flushUserSeries(0xc42030e000, 0xc4200177e8, 0x1, 0x53e6187edce1d0c1, 0x0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:519 +0x20c
github.com/weaveworks/cortex/ingester.(*Ingester).flushLoop(0xc42030e000, 0x31)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:472 +0xe8
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:183 +0x905

goroutine 134 [select]:
github.com/weaveworks/cortex/ingester.(*Ingester).loop(0xc42030e000)
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:387 +0x1ca
created by github.com/weaveworks/cortex/ingester.New
	/go/src/github.com/weaveworks/cortex/ingester/ingester.go:187 +0x963

goroutine 135 [IO wait]:
net.runtime_pollWait(0x7fd2c969ff50, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42023e8b0, 0x72, 0xc420046d28, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42023e8b0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc42023e850, 0x0, 0x10312c0, 0xc45dc14960)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc420160238, 0x29e8d60800, 0x0, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).AcceptTCP(0xc420160238, 0xc420046e50, 0xc420046e58, 0xc420046e48)
	/usr/local/go/src/net/tcpsock.go:209 +0x49
net/http.tcpKeepAliveListener.Accept(0xc420160238, 0xb47818, 0xc4cbab6580, 0x1038500, 0xc420241650)
	/usr/local/go/src/net/http/server.go:2608 +0x2f
net/http.(*Server).Serve(0xc420180e00, 0x1037e80, 0xc420160238, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2273 +0x1ce
net/http.(*Server).ListenAndServe(0xc420180e00, 0xc420180e00, 0x0)
	/usr/local/go/src/net/http/server.go:2219 +0xb4
net/http.ListenAndServe(0xc4201b4508, 0x3, 0x1032a80, 0xc4201b1650, 0x3, 0x10)
	/usr/local/go/src/net/http/server.go:2351 +0xa0
created by github.com/weaveworks/cortex/server.(*Server).Run
	/go/src/github.com/weaveworks/cortex/server/server.go:100 +0x29f

goroutine 136 [IO wait, 51 minutes]:
net.runtime_pollWait(0x7fd2c969fe90, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420300a70, 0x72, 0xc4202e9d70, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420300a70, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420300a10, 0x0, 0x10312c0, 0xc473f85260)
	/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc4201ae028, 0x43356e, 0xc4202e9e20, 0x63e16c)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc4201ae028, 0xb47480, 0xc4202440c0, 0x103ae60, 0xc476182908)
	/usr/local/go/src/net/tcpsock.go:222 +0x49
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve(0xc4202440c0, 0x1037580, 0xc4201ae028, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:368 +0x187
created by github.com/weaveworks/cortex/server.(*Server).Run
	/go/src/github.com/weaveworks/cortex/server/server.go:108 +0x419

goroutine 137 [select, 6829 minutes, locked to thread]:
runtime.gopark(0xb47e10, 0x0, 0xaf5983, 0x6, 0x18, 0x2)
	/usr/local/go/src/runtime/proc.go:259 +0x13a
runtime.selectgoImpl(0xc420328730, 0x0, 0x18)
	/usr/local/go/src/runtime/select.go:423 +0x11d9
runtime.selectgo(0xc420328730)
	/usr/local/go/src/runtime/select.go:238 +0x1c
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal1_unix.go:304 +0x2f3
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1

goroutine 402 [IO wait]:
net.runtime_pollWait(0x7fd2c969fad0, 0x72, 0x7)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4207d5250, 0x72, 0xc4202ec8d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4207d5250, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4207d51f0, 0xc420940000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420c2f338, 0xc420940000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc420192ae0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420192ae0, 0xc420244038, 0x9, 0x9, 0x10, 0xc4fc23dad0, 0x0)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc420192ae0, 0xc420244038, 0x9, 0x9, 0x9, 0xa984a0, 0x402801, 0xc4fc23dad0)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc420192ae0, 0xc420244038, 0x9, 0x9, 0x4000451e4201, 0xfc00000000000004, 0x5)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc420244038, 0x9, 0x9, 0x102f380, 0xc420192ae0, 0x0, 0xc400000000, 0x42a18c, 0xc4202ecbe8)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420244000, 0x400001034e00, 0x400000801, 0x8010000379d, 0x379d00000004)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc42091cde0, 0xc4fc23dad0, 0xc4fc23dad0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420244300, 0xc42091ce70, 0xb474b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202440c0, 0x103ac20, 0xc420244300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202440c0, 0x103ae60, 0xc420c2f338, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202440c0, 0x103ae60, 0xc420c2f338)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 80 [IO wait]:
net.runtime_pollWait(0x7fd2c969fdd0, 0x72, 0x6)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420202840, 0x72, 0xc4202ea8d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420202840, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4202027e0, 0xc42028e000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4201ae040, 0xc42028e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc4202fa9c0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc4202fa9c0, 0xc42000c278, 0x9, 0x9, 0xb4461d, 0xb4461c, 0xc4a21cdb58)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc4202fa9c0, 0xc42000c278, 0x9, 0x9, 0x9, 0xc4202eaae8, 0x451a90, 0xc4235a1f00)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc4202fa9c0, 0xc42000c278, 0x9, 0x9, 0xc549cafbd0, 0x100000000, 0xc4202eab50)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc42000c278, 0x9, 0x9, 0x102f380, 0xc4202fa9c0, 0x0, 0xc400000000, 0xc4202eac80, 0x7f3095)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42000c240, 0x1926, 0x1926, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4201bbd70, 0xc4a42cecc0, 0xc4a42cecc0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42000c300, 0xc4201bbe00, 0xb474b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202440c0, 0x103ac20, 0xc42000c300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202440c0, 0x103ae60, 0xc4201ae040, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202440c0, 0x103ae60, 0xc4201ae040)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 403 [select, 237 minutes]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc420244300)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 140 [IO wait]:
net.runtime_pollWait(0x7fd2c969fd10, 0x72, 0x8)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4200ec1b0, 0x72, 0xc4204978d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4200ec1b0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200ec150, 0xc420598000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420160350, 0xc420598000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc420192120)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420192120, 0xc4205a8038, 0x9, 0x9, 0x43fe80, 0xc45e5aeda0, 0xc4657e3000)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc420192120, 0xc4205a8038, 0x9, 0x9, 0x9, 0xc420497ae8, 0x451a90, 0xc551c34980)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc420192120, 0xc4205a8038, 0x9, 0x9, 0xc45e5aeda0, 0x1, 0x2)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc4205a8038, 0x9, 0x9, 0x102f380, 0xc420192120, 0x0, 0xc400000000, 0xc420497c80, 0x7f3095)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4205a8000, 0x222c, 0x222c, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc42054c1b0, 0xc4b8f52240, 0xc4b8f52240, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4205a80c0, 0xc42054c240, 0xb474b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202440c0, 0x103ac20, 0xc4205a80c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202440c0, 0x103ae60, 0xc420160350, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202440c0, 0x103ae60, 0xc420160350)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 141 [select]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc4205a80c0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 2618 [select, 213 minutes]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc420244900)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 2026 [select, 176 minutes]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc420244780)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 2025 [IO wait]:
net.runtime_pollWait(0x7fd2c969fb90, 0x72, 0xa)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4218f27d0, 0x72, 0xc4202ee8d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4218f27d0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4218f2770, 0xc421e96000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc421e6eb88, 0xc421e96000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc42100d5c0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc42100d5c0, 0xc4202446f8, 0x9, 0x9, 0x10, 0xc504e25270, 0x0)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc42100d5c0, 0xc4202446f8, 0x9, 0x9, 0x9, 0xa984a0, 0x402801, 0xc504e25270)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc42100d5c0, 0xc4202446f8, 0x9, 0x9, 0x4000451e4201, 0xfc00000000000004, 0x5)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc4202446f8, 0x9, 0x9, 0x102f380, 0xc42100d5c0, 0x0, 0xc400000000, 0x42a18c, 0xc4202eebe8)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4202446c0, 0x400001034e00, 0x400000801, 0x801000037b3, 0x37b300000004)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc42123a150, 0xc504e25270, 0xc504e25270, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420244780, 0xc42123a1e0, 0xb474b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202440c0, 0x103ac20, 0xc420244780)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202440c0, 0x103ae60, 0xc421e6eb88, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202440c0, 0x103ae60, 0xc421e6eb88)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 415998451 [IO wait]:
net.runtime_pollWait(0x7fd2918de5d8, 0x72, 0x17)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4bc588300, 0x72, 0xc46e9f86f0, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4bc588300, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4bc5882a0, 0xc51c17e000, 0x1000, 0x1000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc5071da980, 0xc51c17e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc4b7a26810, 0x7fd2c84ff650, 0xc5071da980, 0x5, 0xc5071da980, 0x28)
	/usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc5345caa80, 0xb47e17, 0xc5345cab88, 0x2a)
	/usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc5345caa80, 0xc538c08000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1113 +0x116
net/http.(*persistConn).Read(0xc433c84300, 0xc538c08000, 0x1000, 0x1000, 0x30, 0xc46e9f8b58, 0x43a43c)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc499573500)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc499573500, 0x1, 0x0, 0x1, 0x0, 0xc51f982de0, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc433c84300)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 415998588 [select]:
net/http.(*persistConn).roundTrip(0xc433c84300, 0xc4a60219c0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200d4000, 0xc525886960, 0xc4200d4000, 0x0, 0xc400000000)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc525886960, 0x1030bc0, 0xc4200d4000, 0x0, 0x0, 0x0, 0x8, 0xc513185790, 0xc492aa51e8)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0x10507a0, 0xc525886960, 0x0, 0x0, 0x0, 0xc492aa51e8, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0x10507a0, 0xc525886960, 0xb479d0, 0x3, 0xc513185a01, 0x83d240)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Do(0x10507a0, 0xc525886960, 0xc4a1ad06c0, 0xaf2be7, 0x2)
	/usr/local/go/src/net/http/client.go:187 +0x11d
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/corehandlers.glob..func2(0xc50e59c700)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go:73 +0x53
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request.(*HandlerList).Run(0xc50e59c870, 0xc50e59c700)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go:136 +0x87
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request.(*Request).Send(0xc50e59c700, 0xc548633110, 0xc50e59c700)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request/request.go:268 +0x22c
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/service/s3.(*S3).PutObject(0xc42017c188, 0xc548633110, 0xc4a1ad0570, 0x2f, 0xc4ade90d00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/service/s3/api.go:2897 +0x4d
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunk.func1(0x7fd2c84ff6a0, 0xc4a1ad06f0, 0x1050960, 0xc42015d110)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:380 +0x1b1
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogramStatus(0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xafa423, 0xc, 0xc4201601a0, 0xb47308, 0xc513185e48, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:45 +0x14d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogram(0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xafa423, 0xc, 0xc4201601a0, 0xc513185e48, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:27 +0x70
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunk(0xc42030a000, 0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xc42048ffad, 0x1, 0xc4e3ebb0e0, 0xc42032c760, 0xc42032c770)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:382 +0x127
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks.func1(0xc4aff7d620, 0xc42030a000, 0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xc42048ffad, 0x1, 0xc4a1ad0570, 0x2f, 0x159f4d50f56, 0x159f50260e6, ...)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:353 +0xb9
created by github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:354 +0x16b

goroutine 415998612 [select]:
net/http.(*persistConn).writeLoop(0xc506bcfc00)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 415997862 [IO wait]:
net.runtime_pollWait(0x7fd29d79cf28, 0x72, 0x11)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc506908530, 0x72, 0xc44bbb96f0, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc506908530, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc5069084d0, 0xc43678e000, 0x2000, 0x2000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4f803fb80, 0xc43678e000, 0x2000, 0x2000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc4c7c79e60, 0x7fd2c84ff650, 0xc4f803fb80, 0x5, 0xc4f803fb80, 0x28)
	/usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc4e928c700, 0xb47e17, 0xc4e928c808, 0xc42030c340)
	/usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc4e928c700, 0xc52decc000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1113 +0x116
net/http.(*persistConn).Read(0xc5573c6200, 0xc52decc000, 0x1000, 0x1000, 0x4ba940, 0xc44bbb9b58, 0x404d8d)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4a2675200)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4a2675200, 0x1, 0xc44bbb9bbd, 0x1, 0x0, 0xc470edcb40, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc5573c6200)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 414856166 [select]:
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc454e31b00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:721 +0x6ab
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:148 +0x855

goroutine 415998550 [select]:
net/http.(*persistConn).writeLoop(0xc506bcf600)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 415998479 [IO wait]:
net.runtime_pollWait(0x7fd2c81cbb50, 0x72, 0x12)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4de4b0ca0, 0x72, 0xc5028936f0, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4de4b0ca0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4de4b0c40, 0xc5022e1000, 0x1000, 0x1000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc44e200038, 0xc5022e1000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc4a6293d70, 0x7fd2c84ff650, 0xc44e200038, 0x5, 0xc44e200038, 0xc54cc27da0)
	/usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc51766bc00, 0xb47e17, 0xc51766bd08, 0xc4810e7900)
	/usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc51766bc00, 0xc51c887000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1113 +0x116
net/http.(*persistConn).Read(0xc4e4def200, 0xc51c887000, 0x1000, 0x1000, 0x4b2b13, 0xc478f43b78, 0xc478f43b88)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4446648a0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4446648a0, 0x1, 0xc4bf18c960, 0xc478f43c60, 0x0, 0x0, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4e4def200)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 415994953 [select]:
net/http.(*persistConn).roundTrip(0xc44ef6ca00, 0xc4ee69ba20, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200d4000, 0xc504b52c30, 0xc4200d4000, 0x0, 0xc400000000)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc504b52c30, 0x1030bc0, 0xc4200d4000, 0x0, 0x0, 0x0, 0x8, 0xc54a453790, 0xc4734b7ac8)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0x10507a0, 0xc504b52c30, 0x0, 0x0, 0x0, 0xc4734b7ac8, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0x10507a0, 0xc504b52c30, 0xb479d0, 0x3, 0xc54a453a01, 0x83d240)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Do(0x10507a0, 0xc504b52c30, 0xc525172ea0, 0xaf2be7, 0x2)
	/usr/local/go/src/net/http/client.go:187 +0x11d
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/corehandlers.glob..func2(0xc4d458d880)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go:73 +0x53
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request.(*HandlerList).Run(0xc4d458d9f0, 0xc4d458d880)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go:136 +0x87
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request.(*Request).Send(0xc4d458d880, 0xc4a368f040, 0xc4d458d880)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request/request.go:268 +0x22c
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/service/s3.(*S3).PutObject(0xc42017c188, 0xc4a368f040, 0xc4f45bb140, 0x2f, 0xc531f22a40)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/service/s3/api.go:2897 +0x4d
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunk.func1(0x7fd2c84ff6a0, 0xc525172ed0, 0x1050960, 0xc42015d110)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:380 +0x1b1
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogramStatus(0x7fd2c84ff6a0, 0xc4f45bb0e0, 0xafa423, 0xc, 0xc4201601a0, 0xb47308, 0xc54a453e48, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:45 +0x14d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogram(0x7fd2c84ff6a0, 0xc4f45bb0e0, 0xafa423, 0xc, 0xc4201601a0, 0xc54a453e48, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:27 +0x70
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunk(0xc42030a000, 0x7fd2c84ff6a0, 0xc4f45bb0e0, 0xc4200177e8, 0x1, 0xc53e4a98b0, 0xc4f48ddf60, 0xc4f48ddf70)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:382 +0x127
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks.func1(0xc4f1fc4480, 0xc42030a000, 0x7fd2c84ff6a0, 0xc4f45bb0e0, 0xc4200177e8, 0x1, 0xc4f45bb140, 0x2f, 0x159f431750d, 0x159f469fdbd, ...)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:353 +0xb9
created by github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:354 +0x16b

goroutine 415996974 [select]:
net/http.(*persistConn).writeLoop(0xc4e0d93c00)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 415997863 [select]:
net/http.(*persistConn).writeLoop(0xc5573c6200)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 415998549 [IO wait]:
net.runtime_pollWait(0x7fd29dc2a538, 0x72, 0xf)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4a7941170, 0x72, 0xc4339086f0, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4a7941170, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4a7941110, 0xc53feb5000, 0x400, 0x400, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc492aa47b8, 0xc53feb5000, 0x400, 0x400, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc4825be090, 0x7fd2c84ff650, 0xc492aa47b8, 0x5, 0xc492aa47b8, 0x19)
	/usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc514023180, 0xb47e17, 0xc514023288, 0xa4bf01)
	/usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc514023180, 0xc54925f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1113 +0x116
net/http.(*persistConn).Read(0xc506bcf600, 0xc54925f000, 0x1000, 0x1000, 0x4b2b13, 0xc42097b378, 0xc42097b388)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc47c718720)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc47c718720, 0x1, 0xc47c7187e0, 0xc42097b460, 0x18, 0x0, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc506bcf600)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 415998586 [select]:
net/http.(*persistConn).roundTrip(0xc506bcf600, 0xc4cc17d120, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200d4000, 0xc4fe924870, 0xc4200d4000, 0x0, 0xc400000000)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc4fe924870, 0x1030bc0, 0xc4200d4000, 0x0, 0x0, 0x0, 0x8, 0xc5350c1790, 0xc54df0ae08)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0x10507a0, 0xc4fe924870, 0x0, 0x0, 0x0, 0xc54df0ae08, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0x10507a0, 0xc4fe924870, 0xb479d0, 0x3, 0xc5350c1a01, 0x83d240)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Do(0x10507a0, 0xc4fe924870, 0xc4ab7580f0, 0xaf2be7, 0x2)
	/usr/local/go/src/net/http/client.go:187 +0x11d
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/corehandlers.glob..func2(0xc52efc1180)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go:73 +0x53
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request.(*HandlerList).Run(0xc52efc12f0, 0xc52efc1180)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go:136 +0x87
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request.(*Request).Send(0xc52efc1180, 0xc503beedd0, 0xc52efc1180)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request/request.go:268 +0x22c
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/service/s3.(*S3).PutObject(0xc42017c188, 0xc503beedd0, 0xc4a1ad0510, 0x2f, 0xc4933043c0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/service/s3/api.go:2897 +0x4d
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunk.func1(0x7fd2c84ff6a0, 0xc4ab758120, 0x1050960, 0xc42015d110)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:380 +0x1b1
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogramStatus(0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xafa423, 0xc, 0xc4201601a0, 0xb47308, 0xc5350c1e48, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:45 +0x14d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogram(0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xafa423, 0xc, 0xc4201601a0, 0xc5350c1e48, 0x0, 0x4b472c)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:27 +0x70
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunk(0xc42030a000, 0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xc42048ffad, 0x1, 0xc4e15a76d0, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:382 +0x127
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks.func1(0xc4aff7d620, 0xc42030a000, 0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xc42048ffad, 0x1, 0xc4a1ad0510, 0x2f, 0x159f479f706, 0x159f4a74896, ...)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:353 +0xb9
created by github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:354 +0x16b

goroutine 415998452 [select]:
net/http.(*persistConn).writeLoop(0xc433c84300)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 415994930 [select]:
net/http.(*persistConn).writeLoop(0xc44ef6ca00)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 415998585 [select]:
net/http.(*persistConn).roundTrip(0xc4e4def200, 0xc471ef53c0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200d4000, 0xc5154a8ff0, 0xc4200d4000, 0x0, 0xc400000000)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc5154a8ff0, 0x1030bc0, 0xc4200d4000, 0x0, 0x0, 0x0, 0x8, 0xc532039790, 0xc4845b3628)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0x10507a0, 0xc5154a8ff0, 0x0, 0x0, 0x0, 0xc4845b3628, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0x10507a0, 0xc5154a8ff0, 0xb479d0, 0x3, 0xc532039a01, 0x83d240)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Do(0x10507a0, 0xc5154a8ff0, 0xc42511cb70, 0xaf2be7, 0x2)
	/usr/local/go/src/net/http/client.go:187 +0x11d
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/corehandlers.glob..func2(0xc4851c6e00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go:73 +0x53
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request.(*HandlerList).Run(0xc4851c6f70, 0xc4851c6e00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go:136 +0x87
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request.(*Request).Send(0xc4851c6e00, 0xc4d6f4b520, 0xc4851c6e00)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request/request.go:268 +0x22c
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/service/s3.(*S3).PutObject(0xc42017c188, 0xc4d6f4b520, 0xc4a1ad04e0, 0x2f, 0xc4b425b140)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/service/s3/api.go:2897 +0x4d
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunk.func1(0x7fd2c84ff6a0, 0xc42511cba0, 0x1050960, 0xc42015d110)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:380 +0x1b1
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogramStatus(0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xafa423, 0xc, 0xc4201601a0, 0xb47308, 0xc532039e48, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:45 +0x14d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogram(0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xafa423, 0xc, 0xc4201601a0, 0xc532039e48, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:27 +0x70
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunk(0xc42030a000, 0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xc42048ffad, 0x1, 0xc51678a230, 0xc539a6ff60, 0xc539a6ff70)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:382 +0x127
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks.func1(0xc4aff7d620, 0xc42030a000, 0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xc42048ffad, 0x1, 0xc4a1ad04e0, 0x2f, 0x159f4588c56, 0x159f479bc6e, ...)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:353 +0xb9
created by github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:354 +0x16b

goroutine 415998326 [IO wait]:
net.runtime_pollWait(0x7fd29d79d9a8, 0x72, 0xe)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4cefce3e0, 0x72, 0xc47f2936f0, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4cefce3e0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4cefce380, 0xc52380e000, 0x1000, 0x1000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc5283450c0, 0xc52380e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc4c560c750, 0x7fd2c84ff650, 0xc5283450c0, 0x5, 0xc5283450c0, 0x28)
	/usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc47a97b180, 0xb47e17, 0xc47a97b288, 0xc42030da00)
	/usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc47a97b180, 0xc52fc1f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1113 +0x116
net/http.(*persistConn).Read(0xc483f5f900, 0xc52fc1f000, 0x1000, 0x1000, 0x4ba940, 0xc47f293b58, 0x404d8d)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc476041c20)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc476041c20, 0x1, 0xc47f293bbd, 0x1, 0x0, 0xc486a18660, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc483f5f900)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 415998611 [IO wait]:
net.runtime_pollWait(0x7fd2919201e8, 0x72, 0x14)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4c5a10df0, 0x72, 0xc46a3686f0, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4c5a10df0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4c5a10d90, 0xc51c886000, 0x1000, 0x1000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc54df0ae38, 0xc51c886000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc4ab759650, 0x7fd2c84ff650, 0xc54df0ae38, 0x5, 0xc54df0ae38, 0x19)
	/usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc52efc1500, 0xb47e17, 0xc52efc1608, 0xa4bf01)
	/usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc52efc1500, 0xc48b59a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1113 +0x116
net/http.(*persistConn).Read(0xc506bcfc00, 0xc48b59a000, 0x1000, 0x1000, 0x4b2b13, 0xc478f43b78, 0xc478f43b88)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4bf18ca20)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4bf18ca20, 0x1, 0xc444664ba0, 0xc478f43c60, 0x0, 0x0, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc506bcfc00)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 415994097 [IO wait]:
net.runtime_pollWait(0x7fd29dbf67f0, 0x72, 0x1a)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc53e8b8b50, 0x72, 0xc46a3696f0, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc53e8b8b50, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc53e8b8af0, 0xc50f35e000, 0x1000, 0x1000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc53e227590, 0xc50f35e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
crypto/tls.(*block).readFromUntil(0xc52b9499b0, 0x7fd2c84ff650, 0xc53e227590, 0x5, 0xc53e227590, 0x19)
	/usr/local/go/src/crypto/tls/conn.go:476 +0x91
crypto/tls.(*Conn).readRecord(0xc472fa7180, 0xb47e17, 0xc472fa7288, 0xa4bf01)
	/usr/local/go/src/crypto/tls/conn.go:578 +0xc4
crypto/tls.(*Conn).Read(0xc472fa7180, 0xc531318000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1113 +0x116
net/http.(*persistConn).Read(0xc44ef6ca00, 0xc531318000, 0x1000, 0x1000, 0x4b2b13, 0xc51856fb78, 0xc51856fb88)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc47e57e8a0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc47e57e8a0, 0x1, 0xc47e57e900, 0xc51856fc60, 0x0, 0x0, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc44ef6ca00)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

goroutine 414856165 [IO wait]:
net.runtime_pollWait(0x7fd2c848e388, 0x72, 0xc)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc463125c60, 0x72, 0xc5028908d8, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc463125c60, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc463125c00, 0xc4ed796000, 0x8000, 0x8000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc476182908, 0xc4ed796000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc4e6fd15c0)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc4e6fd15c0, 0xc454e31a78, 0x9, 0x9, 0x43fe80, 0xc5150e8000, 0x0)
	/usr/local/go/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x102f380, 0xc4e6fd15c0, 0xc454e31a78, 0x9, 0x9, 0x9, 0xc502890ae8, 0x451a90, 0xc5321b9840)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x102f380, 0xc4e6fd15c0, 0xc454e31a78, 0x9, 0x9, 0xc5150e8000, 0x0, 0x1)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.readFrameHeader(0xc454e31a78, 0x9, 0x9, 0x102f380, 0xc4e6fd15c0, 0x0, 0xc500000000, 0xc502890c80, 0x7f3095)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/weaveworks/cortex/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc454e31a40, 0x99, 0x99, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/golang.org/x/net/http2/frame.go:464 +0xa4
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4d89e3920, 0xc53ad98780, 0xc53ad98780, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc454e31b00, 0xc4d89e39b0, 0xb474b8)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/transport/http2_server.go:293 +0x200
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4202440c0, 0x103ac20, 0xc454e31b00)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:476 +0x16c
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4202440c0, 0x103ae60, 0xc476182908, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:457 +0x416
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4202440c0, 0x103ae60, 0xc476182908)
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:429 +0x470
created by github.com/weaveworks/cortex/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/server.go:398 +0x5ac

goroutine 415998587 [select]:
net/http.(*persistConn).roundTrip(0xc506bcfc00, 0xc485b49fa0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200d4000, 0xc519578d20, 0xc4200d4000, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc519578d20, 0x1030bc0, 0xc4200d4000, 0x0, 0x0, 0x0, 0x8, 0xc53258b790, 0xc44e200000)
	/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0x10507a0, 0xc519578d20, 0x0, 0x0, 0x0, 0xc44e200000, 0x0, 0x1)
	/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0x10507a0, 0xc519578d20, 0xb479d0, 0x3, 0xc53258ba01, 0x83d240)
	/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Do(0x10507a0, 0xc519578d20, 0xc4a6292390, 0xaf2be7, 0x2)
	/usr/local/go/src/net/http/client.go:187 +0x11d
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/corehandlers.glob..func2(0xc51766b880)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go:73 +0x53
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request.(*HandlerList).Run(0xc51766b9f0, 0xc51766b880)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go:136 +0x87
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request.(*Request).Send(0xc51766b880, 0xc531c7add0, 0xc51766b880)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/aws/request/request.go:268 +0x22c
github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/service/s3.(*S3).PutObject(0xc42017c188, 0xc531c7add0, 0xc4a1ad0540, 0x2f, 0xc4a3273440)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/aws/aws-sdk-go/service/s3/api.go:2897 +0x4d
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunk.func1(0x7fd2c84ff6a0, 0xc4a62923f0, 0x1050960, 0xc42015d110)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:380 +0x1b1
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogramStatus(0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xafa423, 0xc, 0xc4201601a0, 0xb47308, 0xc53258be48, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:45 +0x14d
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogram(0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xafa423, 0xc, 0xc4201601a0, 0xc53258be48, 0x0, 0x0)
	/go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:27 +0x70
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunk(0xc42030a000, 0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xc42048ffad, 0x1, 0xc521402c30, 0xc4204c0f60, 0xc4204c0f70)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:382 +0x127
github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks.func1(0xc4aff7d620, 0xc42030a000, 0x7fd2c84ff6a0, 0xc4a1ad04b0, 0xc42048ffad, 0x1, 0xc4a1ad0540, 0x2f, 0x159f4a7832e, 0x159f4d4d4be, ...)
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:353 +0xb9
created by github.com/weaveworks/cortex/chunk.(*AWSStore).putChunks
	/go/src/github.com/weaveworks/cortex/chunk/chunk_store.go:354 +0x16b

goroutine 415998327 [select]:
net/http.(*persistConn).writeLoop(0xc483f5f900)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 415998480 [select]:
net/http.(*persistConn).writeLoop(0xc4e4def200)
	/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1063 +0x50e

goroutine 415996973 [IO wait]:
net.runtime_pollWait(0x7fd29d86d6a0, 0x72, 0x13)
	/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc507bc93a0, 0x72, 0xc46b6519d0, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc507bc93a0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc507bc9340, 0xc46b379000, 0x1000, 0x1000, 0x0, 0x1033540, 0xc420016150)
	/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc51ab885b8, 0xc46b379000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc4e0d93c00, 0xc46b379000, 0x1000, 0x1000, 0x4ba940, 0xc46b651b58, 0x404d8d)
	/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4bfbddd40)
	/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4bfbddd40, 0x1, 0xc46b651bbd, 0x1, 0x0, 0xc4fdfb8c00, 0x0)
	/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc4e0d93c00)
	/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1062 +0x4e9

*** end