Created
February 1, 2021 00:03
-
-
Save amckinley/eea8e5617cdf687524cca326670b78b9 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| level=warn ts=2021-01-31T23:53:11.343049641Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EXAHJGKSGFTQ75X5FNNPP3SK:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:53:11.343138289Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=1 firstKey=S:01EX9P3FDZ055JCKZ7ZS22ECMN:41603280 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:53:11.343198509Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EX9P34379RNTDWJMJT3KC8H4:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:53:11.343225582Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EXARD9S52S833EDTV1N4PNM6:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:53:11.343228637Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EXARCPWGCY1ZEFYM92ZW7RC6:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:53:11.343260751Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EX91G2356WDQ8W550ESD8F1J:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:53:11.343286865Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EX9F67HNVEJASGS6D2NXY000:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:53:11.343297522Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX9F7493XY5PF8MD5E3D7A0B:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:53:11.343250871Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EX9P34FVZRA87M87JCC2TJ75:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:53:11.343348809Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EXAHHZ55RKBK2B7S0829S6JP:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:53:11.343393875Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=7 firstKey=P:01EX98C4PQHSAWQPJEDMY95H46:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:53:11.343432426Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX98B4TJTCSPNNZ8ZPMV7PZS:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:53:11.343470569Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EXAZ9E6BQJ6A81XWHJQAZEP1:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:53:11.343462462Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EX9F6J47GDF5D3G0TG4B8GWA:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=info ts=2021-01-31T23:53:11.376494137Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=20.803172828s id=01ESSEX8EDA4STFQV1JJD7EMR5 | |
| level=warn ts=2021-01-31T23:53:11.390090352Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=242.504939ms err="rpc error: code = Aborted desc = fetch series for block 01EX1R0NTGXESM004QECZJJ8GC: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.956959115Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:11.957147646Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:11.957149678Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:11.95728205Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:11.957296701Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:11.957361595Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:11.957365621Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:11.957363421Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:11.957621888Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=807.675587ms err="rpc error: code = Aborted desc = fetch series for block 01EX61BF90G6YSHK77S0C9A9WJ: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.957631992Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=22.32530398s err="rpc error: code = Unknown desc = send series response hints: rpc error: code = Internal desc = transport: transport: the stream is done or WriteHeader was already called" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.957627058Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=18.762633565s err="rpc error: code = Unknown desc = send series response hints: rpc error: code = Internal desc = transport: transport: the stream is done or WriteHeader was already called" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.957656695Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.694562044s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.95777634Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=18.789148222s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.957803274Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=22.281748961s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.957832084Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=22.282431546s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.957847276Z caller=grpc_logging.go:55 duration=18.762546676s method=/gatewaypb.StoreGateway/Series err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.957855371Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.695489724s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.957867812Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=807.850912ms err="rpc error: code = Aborted desc = fetch series for block 01EX5TGKRTC4CH9FAMT8QJQTM4: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.957868609Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.694784205s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.957875221Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=819.712489ms err="rpc error: code = Aborted desc = fetch series for block 01EX8KQAPJWAXG2QW7GJ6W7KPC: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.957891925Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=11.105779437s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.958031279Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.694568108s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.958048866Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.695648799s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:11.958054201Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.694870885s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.368025466Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVAKB2P94Y7MG1GTRYQD4TNR:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:12.36805452Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVAKB2P94Y7MG1GTRYQD4TNR:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context deadline exceeded" | |
| level=warn ts=2021-01-31T23:53:12.368079157Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVAKB2P94Y7MG1GTRYQD4TNR:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context deadline exceeded" | |
| level=warn ts=2021-01-31T23:53:12.36817648Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVAKB2P94Y7MG1GTRYQD4TNR:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:12.368285483Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m33.873592935s err="rpc error: code = Aborted desc = fetch series for block 01EV93950R0B8D15VJCEXVPM27: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.368449431Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=2m4.050955267s err="rpc error: code = Aborted desc = fetch series for block 01EV939W5K31H36XH9PC4Z0HZF: expanded matching posting: get postings: read postings range: read range: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.36844958Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m27.047762389s err="rpc error: code = Aborted desc = fetch series for block 01EV9A439HJAKPZ0JJDGZBZDYQ: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.368472593Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=3m28.442545424s err="rpc error: code = Aborted desc = fetch series for block 01EV939W5K31H36XH9PC4Z0HZF: expanded matching posting: get postings: read postings range: get range reader: context deadline exceeded" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.36856347Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=1m3.974794111s err="rpc error: code = Aborted desc = fetch series for block 01EV80XRBJ9WVTV9YH54VCXGZS: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.369077641Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m18.661856529s err="rpc error: code = Aborted desc = fetch series for block 01EV8ENZQD6D1Z48NKHX8J6SHD: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.429636435Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVS68N7C7T5RW7XQ5A07M44C:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:12.429893942Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m29.314522431s err="rpc error: code = Aborted desc = fetch series for block 01EVQFAFWZP1MJ4Z19SMS08271: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.42997345Z caller=grpc_logging.go:55 duration=4m33.928759114s method=/gatewaypb.StoreGateway/Series err="rpc error: code = Aborted desc = fetch series for block 01EVQFAFWZP1MJ4Z19SMS08271: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.430102172Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=1m4.03314589s err="rpc error: code = Aborted desc = fetch series for block 01EVQP6MRD00T92DXGRFJWCQKX: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.430811972Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=1m4.033891552s err="rpc error: code = Aborted desc = fetch series for block 01EVQP6MRD00T92DXGRFJWCQKX: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.444550963Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX5KK5RF994VZEKPD4Y1CX9G:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:12.444611736Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=22.771197618s err="rpc error: code = Unknown desc = send series response hints: rpc error: code = Internal desc = transport: transport: the stream is done or WriteHeader was already called" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.445102977Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.276439592s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.445462757Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=22.801971193s err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.445610833Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=20.183118048s err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.445609705Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=20.183138967s err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.445635043Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.250272442s err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.445732312Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=20.182729644s err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.445731068Z caller=grpc_logging.go:55 duration=20.182797936s method=/gatewaypb.StoreGateway/Series err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.524130631Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m27.307749729s err="rpc error: code = Aborted desc = fetch series for block 01EVYB2AHNA5KY6AETVAG7TC4W: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.524222444Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m34.027191935s err="rpc error: code = Aborted desc = fetch series for block 01EVXXAK4C8Y95A19X4YT8WF1Q: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.966626754Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m34.472055433s err="rpc error: code = Aborted desc = fetch series for block 01EV6QRT1C9AVYVV716F7ABK8K: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:12.966744099Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m28.986864233s err="rpc error: code = Aborted desc = fetch series for block 01EV6QRT1C9AVYVV716F7ABK8K: expanded matching posting: get postings: read postings range: get range reader: context deadline exceeded" msg="gRPC\n" | |
| level=info ts=2021-01-31T23:53:12.974596472Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=24.051961462s id=01ETQ2GTR4G87CZDCSDB8T09W1 | |
| level=warn ts=2021-01-31T23:53:13.471464729Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m30.342884486s err="rpc error: code = Aborted desc = fetch series for block 01EVHWSYD41MS3ZDBA2A7TMF18: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:13.471460547Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m34.976754391s err="rpc error: code = Aborted desc = fetch series for block 01EVHWSYD41MS3ZDBA2A7TMF18: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:13.471507787Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=3m46.42793758s err="rpc error: code = Aborted desc = fetch series for block 01EVFZ0P60KPQPWA4FZTT55XA6: expanded matching posting: get postings: read postings range: get range reader: context deadline exceeded" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:13.489283002Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m12.010613701s err="rpc error: code = Aborted desc = fetch series for block 01EVGKJQ2HN118ENPB1292X5QN: expanded matching posting: get postings: read postings range: get range reader: context deadline exceeded" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:14.196148666Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVZV3SC6K3SNZ5BWDCN6M03V:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:14.196247146Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVZV3SC6K3SNZ5BWDCN6M03V:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context deadline exceeded" | |
| level=warn ts=2021-01-31T23:53:14.221601519Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m20.513910043s err="rpc error: code = Aborted desc = fetch series for block 01EVM8BJSB72C45NCV6XSX2H1M: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:14.221598817Z caller=grpc_logging.go:55 duration=4m28.964798359s method=/gatewaypb.StoreGateway/Series err="rpc error: code = Aborted desc = fetch series for block 01EVJAGST35SFJ7SH6GWR40JZN: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:14.221634747Z caller=grpc_logging.go:55 duration=4m35.727079755s method=/gatewaypb.StoreGateway/Series err="rpc error: code = Aborted desc = fetch series for block 01EVJAGST35SFJ7SH6GWR40JZN: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:14.597463325Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=2m21.116840928s err="rpc error: code = Aborted desc = fetch series for block 01EV0GM3R428SGPDEM4VRTMZ9E: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:15.564493246Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVENSB46YZVQA71QBG0WQS8K:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:15.604835234Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m37.109816104s err="rpc error: code = Aborted desc = fetch series for block 01EW1RY6GNHPDJMRTACF9SFF84: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:15.60485295Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=2m22.12426968s err="rpc error: code = Aborted desc = fetch series for block 01EW1RY6GNHPDJMRTACF9SFF84: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:15.604857486Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=1m7.210766342s err="rpc error: code = Aborted desc = fetch series for block 01EW1RY6GNHPDJMRTACF9SFF84: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:15.604853075Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m30.3882386s err="rpc error: code = Aborted desc = fetch series for block 01EW1RY6GNHPDJMRTACF9SFF84: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:15.604941305Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVZV4X2SH4JVME81NT5BJGMA:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:15.605278644Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m21.895793234s err="rpc error: code = Aborted desc = fetch series for block 01EW1RY6GNHPDJMRTACF9SFF84: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:15.805887821Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVZV4X2SH4JVME81NT5BJGMA:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="read tcp 10.11.134.125:48666->10.11.105.37:11211: i/o timeout" | |
| level=warn ts=2021-01-31T23:53:15.806459803Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=3m31.880567966s err="rpc error: code = Aborted desc = fetch series for block 01EW0XEGE3NP7M550X1ASAKF2Y: expanded matching posting: get postings: read postings range: get range reader: context deadline exceeded" msg="gRPC\n" | |
| level=info ts=2021-01-31T23:53:16.368519122Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.850008854s id=01EWN2XC88AF6BNJVAZK3FQEHA | |
| level=info ts=2021-01-31T23:53:16.370621093Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.871352225s id=01EW1B5N726XJ01V8RKBGJ3WKA | |
| level=info ts=2021-01-31T23:53:16.372195988Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.1733649s id=01EVR3YN9BRV19TKEWK31YSK4A | |
| level=info ts=2021-01-31T23:53:16.373583139Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.854518142s id=01ESS15PM6QM0KWSMWVKT2MKC2 | |
| level=info ts=2021-01-31T23:53:16.375052299Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.189782239s id=01EXAZ9F7C5H5JNCW5CHESTQEH | |
| level=info ts=2021-01-31T23:53:16.376429517Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.174933994s id=01ETSMWHZFPRNTNHSY4VS0WKJQ | |
| level=info ts=2021-01-31T23:53:16.377959104Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.176485971s id=01EWPSV69JHTHMZXCC7FKBD046 | |
| level=info ts=2021-01-31T23:53:16.37953622Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.197427573s id=01ERM45BDSFDPJEHKEHF6S2320 | |
| level=info ts=2021-01-31T23:53:16.462119558Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.944370371s id=01ESWNYAS0FZ10ZRYPQ8QWC2KS | |
| level=info ts=2021-01-31T23:53:16.463606979Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.264748886s id=01EWT0SZYH058F5VAKV0ZZ4YQ4 | |
| level=info ts=2021-01-31T23:53:16.465615711Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.266803313s id=01ERVMFMGFQ9047AMQFZ8PCX0Q | |
| level=info ts=2021-01-31T23:53:16.466969538Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.949567432s id=01EVM1F2SMHDS7HAAT2S88PEXZ | |
| level=info ts=2021-01-31T23:53:16.468466576Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.971635469s id=01EVJHC9DAGP3VEAV809T9CBSR | |
| level=info ts=2021-01-31T23:53:16.46995367Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.95093276s id=01EW9G3VD1RH1H884SQV81VBJ3 | |
| level=info ts=2021-01-31T23:53:16.471370434Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.267142161s id=01EVNRD1G48N1X19M5NPNMFPG8 | |
| level=info ts=2021-01-31T23:53:16.472806998Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.953985647s id=01EVWM3EPHZ3N4543KH26C5WRD | |
| level=info ts=2021-01-31T23:53:16.474242425Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.981087583s id=01EVCQZNNA13EV2XYDRQ6BM35R | |
| level=warn ts=2021-01-31T23:53:16.533088804Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m38.037829014s err="rpc error: code = Aborted desc = fetch series for block 01EVPCZVMZGYJNZH31TN5GSM58: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:16.702699162Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m22.993018585s err="rpc error: code = Aborted desc = fetch series for block 01EV4SYEY94X7X8VV9H15M7WM2: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:16.703343633Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m38.208539352s err="rpc error: code = Aborted desc = fetch series for block 01EV1C3BTTTJC9ZCGQYCMJ7EE2: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=info ts=2021-01-31T23:53:16.739538771Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=5.363009721s id=01ERZPYP84QZZMBVTCW2T6V32M | |
| level=info ts=2021-01-31T23:53:16.888458112Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.686983992s id=01ERS2360PXK9MW97ZV4N5ZC0E | |
| unexpected fault address 0x7f4c11e8dc72 | |
| fatal error: fault | |
| [signal SIGSEGV: segmentation violation code=0x1 addr=0x7f4c11e8dc72 pc=0x40205f] | |
| goroutine 4498534 [running]: | |
| runtime.throw(0x2533220, 0x5) | |
| /usr/local/go/src/runtime/panic.go:1116 +0x72 fp=0xc13e42cf38 sp=0xc13e42cf08 pc=0x434de2 | |
| runtime.sigpanic() | |
| /usr/local/go/src/runtime/signal_unix.go:702 +0x3cc fp=0xc13e42cf68 sp=0xc13e42cf38 pc=0x44bb8c | |
| cmpbody() | |
| /usr/local/go/src/internal/bytealg/compare_amd64.s:84 +0x9f fp=0xc13e42cf70 sp=0xc13e42cf68 pc=0x40205f | |
| github.com/thanos-io/thanos/pkg/block/indexheader.BinaryReader.postingsOffset(0x2b73f00, 0xc1e84a0760, 0xc162eed9d0, 0x2b44ce0, 0xc1e84a0740, 0xc19d6c31a0, 0x0, 0xc1f2794d40, 0xc2574b6d20, 0xc9e3ffeed0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:696 +0xe54 fp=0xc13e42d0e8 sp=0xc13e42cf70 pc=0x18f5b94 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.BinaryReader.PostingsOffset(0x2b73f00, 0xc1e84a0760, 0xc162eed9d0, 0x2b44ce0, 0xc1e84a0740, 0xc19d6c31a0, 0x0, 0xc1f2794d40, 0xc2574b6d20, 0xc9e3ffeed0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:646 +0xbf fp=0xc13e42d1c8 sp=0xc13e42d0e8 pc=0x18f4b6f | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c418630, 0xc17f393f19, 0x3, 0x7f4c11e8dc72, 0xe, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:162 +0x1b3 fp=0xc13e42d318 sp=0xc13e42d1c8 pc=0x18f7db3 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1ec83cec0, 0xc0522db040, 0x6, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 fp=0xc13e42d578 sp=0xc13e42d318 pc=0x199db71 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1ec83cec0, 0xc1e8721b60, 0x2, 0x2, 0x216f320, 0x3f99388, 0x0, 0x0, 0xbffdee5d4730877c) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac fp=0xc13e42d818 sp=0xc13e42d578 pc=0x199b68c | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc004d1b1a0, 0xc1ec83cec0, 0xc1d7a5ed80, 0xc1e8721b60, 0x2, 0x2, 0xc1ea4afe60, 0x2b44f80, 0xc217e2e2d0, 0xc00017c0b8, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a fp=0xc13e42dd00 sp=0xc13e42d818 pc=0x19931ba | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc08f8eaa20, 0xc1eb7e68c0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe fp=0xc13e42df78 sp=0xc13e42dd00 pc=0x19a373e | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc217e2e2a0, 0xc23e3532d0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 fp=0xc13e42dfd0 sp=0xc13e42df78 pc=0x997bc9 | |
| runtime.goexit() | |
| /usr/local/go/src/runtime/asm_amd64.s:1373 +0x1 fp=0xc13e42dfd8 sp=0xc13e42dfd0 pc=0x467ca1 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 1 [select, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*Manager).AwaitStopped(0xc000481980, 0x2b99cc0, 0xc00005c488, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/util/services/manager.go:145 +0xbb | |
| github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run(0xc000ba9500, 0xc000b25d40, 0x4) | |
| /__w/cortex/cortex/pkg/cortex/cortex.go:416 +0x91c | |
| main.main() | |
| /__w/cortex/cortex/cmd/cortex/main.go:189 +0xe3b | |
| goroutine 4513530 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0e90c8780, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc18aab57a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc078bd3200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4578956 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc079945980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4578953 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f599ab30a80, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00eda7c98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00eda7c80, 0xc1ace34000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc00eda7c80, 0xc1ace34000, 0x8000, 0x8000, 0x10401, 0x0, 0x1040102ade864) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0059b2508, 0xc1ace34000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1ea500f00, 0xc09ba61a78, 0x9, 0x9, 0xc001c90700, 0x7f63e74250f8, 0xc00044a780) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1ea500f00, 0xc09ba61a78, 0x9, 0x9, 0x9, 0xa6fc25, 0xc1931cba40, 0xc1931c0004) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc09ba61a78, 0x9, 0x9, 0x2b41140, 0xc1ea500f00, 0x0, 0x0, 0xc1931cba40, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc09ba61a40, 0xc1931cba40, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc079945680, 0xc19a278a80, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc079945680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc079945680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 307 [chan receive]: | |
| go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc00048c7c0) | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5 | |
| created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80 | |
| goroutine 92 [select]: | |
| go.opencensus.io/stats/view.(*worker).start(0xc0007d4c00) | |
| /__w/cortex/cortex/vendor/go.opencensus.io/stats/view/worker.go:276 +0x100 | |
| created by go.opencensus.io/stats/view.init.0 | |
| /__w/cortex/cortex/vendor/go.opencensus.io/stats/view/worker.go:34 +0x68 | |
| goroutine 161 [chan receive, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/alertmanager.init.0.func1() | |
| /__w/cortex/cortex/pkg/alertmanager/alertmanager.go:95 +0x55 | |
| created by github.com/cortexproject/cortex/pkg/alertmanager.init.0 | |
| /__w/cortex/cortex/pkg/alertmanager/alertmanager.go:91 +0x35 | |
| goroutine 268 [chan receive]: | |
| go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc0006c7100) | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5 | |
| created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80 | |
| goroutine 304 [chan receive]: | |
| go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc00048c500) | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5 | |
| created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80 | |
| goroutine 471 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).resolveAddrsLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:541 +0x136 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:313 +0x1549 | |
| goroutine 386 [select]: | |
| github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollControllerWithTicker(0xc0005c0a90, 0xc00045c730) | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:144 +0xd0 | |
| github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollController(0xc0005c0a90) | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:139 +0x6e | |
| created by github.com/uber/jaeger-client-go.NewRemotelyControlledSampler | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:86 +0x119 | |
| goroutine 378 [select]: | |
| github.com/uber/jaeger-client-go/utils.(*reconnectingUDPConn).reconnectLoop(0xc0005b7110, 0x6fc23ac00) | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:70 +0xed | |
| created by github.com/uber/jaeger-client-go/utils.newReconnectingUDPConn | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:60 +0x10f | |
| goroutine 379 [select]: | |
| github.com/uber/jaeger-client-go.(*remoteReporter).processQueue(0xc000b1fb00) | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/reporter.go:296 +0x120 | |
| created by github.com/uber/jaeger-client-go.NewRemoteReporter | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/reporter.go:237 +0x1a0 | |
| goroutine 472 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 473 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 474 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 475 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 476 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 477 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 478 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 479 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 480 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 481 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 482 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 483 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 484 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 485 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 486 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 487 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 488 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 489 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 490 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 491 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 492 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 493 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 494 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 495 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 496 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 497 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 498 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 499 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 500 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 501 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 502 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 503 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 504 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 505 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 506 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 507 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 508 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 509 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 510 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 511 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 512 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 513 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 514 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 515 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 516 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 517 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 518 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 519 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 520 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 521 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 580 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).resolveAddrsLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:541 +0x136 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:313 +0x1549 | |
| goroutine 584 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 585 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 581 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 583 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 582 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 586 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 587 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 588 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 589 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 590 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 591 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 592 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 609 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 610 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 611 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 612 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 613 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 614 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 615 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 616 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 617 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 618 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 619 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 620 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 621 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 622 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 623 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 624 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 625 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 626 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 627 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 628 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 629 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 630 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 631 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 632 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 633 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 634 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 635 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 636 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 637 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 638 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 639 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 640 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 641 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 642 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 643 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 644 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 645 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 646 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 743 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).resolveAddrsLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:541 +0x136 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:313 +0x1549 | |
| goroutine 745 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 744 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 746 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 747 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 748 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 749 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 750 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 751 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 752 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 753 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 754 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 755 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 756 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 757 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 758 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 759 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 760 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 761 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 762 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 763 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 764 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 765 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 766 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 767 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 768 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 769 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 770 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 771 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 772 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 773 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 774 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 775 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 776 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 777 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 778 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 779 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 780 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 781 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 782 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 783 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 784 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 785 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 786 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 787 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 788 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 789 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 790 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 791 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 792 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 793 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 222 [chan receive, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc0004819e0, 0x2ba22c0, 0xc0001a5140) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 223 [chan receive, 23 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000481a40, 0x2ba22c0, 0xc0001a51c0) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 224 [chan receive, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000481aa0, 0x2ba22c0, 0xc0001a5260) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 801 [chan receive, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000481b00, 0x2ba22c0, 0xc0001a52c0) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 802 [chan receive, 23 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener.func1(0xc000481bc0, 0x2b7fac0, 0xc0001a5380) | |
| /__w/cortex/cortex/pkg/util/services/manager.go:244 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener | |
| /__w/cortex/cortex/pkg/util/services/manager.go:243 +0xfb | |
| goroutine 803 [select, 24 minutes]: | |
| github.com/weaveworks/common/signals.(*Handler).Loop(0xc000845290) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/signals/signals.go:48 +0x1d7 | |
| github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run.func4(0xc000845290, 0xc000481980) | |
| /__w/cortex/cortex/pkg/cortex/cortex.go:405 +0x2b | |
| created by github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run | |
| /__w/cortex/cortex/pkg/cortex/cortex.go:404 +0x71b | |
| goroutine 804 [select, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc0006d0870, 0x2b99c80, 0xc000216c00, 0x4, 0xc0009dcae0, 0xc000e81001, 0xc000898e88) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc0006d0870, 0x2b99c80, 0xc000216c00, 0xc0004819e0, 0x26b3980) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51 | |
| github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc000bf8000, 0x2b99c80, 0xc000216c00, 0x26b85c8, 0xc0006d0918) | |
| /__w/cortex/cortex/pkg/util/module_service.go:70 +0x48 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0900) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 805 [select, 23 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc000a0d440, 0x2b99c80, 0xc000216c80, 0x4, 0xc00017c540, 0x1, 0xc000893e88) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc000a0d440, 0x2b99c80, 0xc000216c80, 0xc000481a40, 0x26b3980) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51 | |
| github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc000216ac0, 0x2b99c80, 0xc000216c80, 0x26b85c8, 0xc000a0d4e8) | |
| /__w/cortex/cortex/pkg/util/module_service.go:70 +0x48 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000a0d4d0) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 806 [select, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc0006d05a0, 0x2b99c80, 0xc000216d80, 0x4, 0xc0009dc6c0, 0xc000e81001, 0xc0009b3e88) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc0006d05a0, 0x2b99c80, 0xc000216d80, 0xc000481aa0, 0x26b3980) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51 | |
| github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc000b25e00, 0x2b99c80, 0xc000216d80, 0x26b85c8, 0xc0006d0648) | |
| /__w/cortex/cortex/pkg/util/module_service.go:70 +0x48 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0630) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 807 [select, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc0006d06c0, 0x2b99c80, 0xc000216e00, 0x4, 0xc0009dc960, 0xc000481b00, 0x26b3980) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc0006d06c0, 0x2b99c80, 0xc000216e00, 0x0, 0xc0006d0750) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51 | |
| github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc000b25f80, 0x2b99c80, 0xc000216e00, 0x26b85c8, 0xc0006d0768) | |
| /__w/cortex/cortex/pkg/util/module_service.go:70 +0x48 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0750) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 808 [select, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1(0x2b99c80, 0xc000217e00, 0x2, 0xc00103c768) | |
| /__w/cortex/cortex/pkg/cortex/server_service.go:28 +0xfb | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d06c0) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 809 [chan receive, 24 minutes]: | |
| github.com/weaveworks/common/server.(*Server).Run(0xc000b262c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:414 +0x15b | |
| github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1.1(0xc000bb0060, 0xc000b262c0) | |
| /__w/cortex/cortex/pkg/cortex/server_service.go:25 +0x57 | |
| created by github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1 | |
| /__w/cortex/cortex/pkg/cortex/server_service.go:23 +0x5e | |
| goroutine 810 [chan receive, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/cortex.ignoreSignalHandler.Loop(0xc0009dc7e0) | |
| /__w/cortex/cortex/pkg/cortex/server_service.go:65 +0x34 | |
| github.com/weaveworks/common/server.(*Server).Run.func1(0xc000b262c0, 0xc000481d40) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:374 +0x3a | |
| created by github.com/weaveworks/common/server.(*Server).Run | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:373 +0x6b | |
| goroutine 811 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f63c073dd58, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0002f2398, 0x72, 0x0, 0x0, 0x253657d) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc0002f2380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1d4 | |
| net.(*netFD).accept(0xc0002f2380, 0xe853b8885b027f48, 0x0, 0xe853b8885b027f48) | |
| /usr/local/go/src/net/fd_unix.go:238 +0x42 | |
| net.(*TCPListener).accept(0xc00080fc20, 0x601742e8, 0xc000a00e08, 0x4db7f6) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc00080fc20, 0xc000a00e58, 0x18, 0xc000e81980, 0x8156cc) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x64 | |
| net/http.(*Server).Serve(0xc000bf6000, 0x2b87e80, 0xc00080fc20, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2930 +0x25d | |
| github.com/weaveworks/common/server.(*Server).Run.func2(0xc000b262c0, 0xc000481d40) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:384 +0x126 | |
| created by github.com/weaveworks/common/server.(*Server).Run | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:381 +0x97 | |
| goroutine 812 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f63c073dc78, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0002f2518, 0x72, 0x0, 0x0, 0x253657d) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc0002f2500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1d4 | |
| net.(*netFD).accept(0xc0002f2500, 0x37e11d600, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:238 +0x42 | |
| net.(*TCPListener).accept(0xc00080fc40, 0xc001042e50, 0xc001042e58, 0x18) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc00080fc40, 0x26b6670, 0xc0002c3380, 0x2bc4400, 0xc00a33bf90) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x64 | |
| google.golang.org/grpc.(*Server).Serve(0xc0002c3380, 0x2b87e80, 0xc00080fc40, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:621 +0x210 | |
| github.com/weaveworks/common/server.(*Server).Run.func3(0xc000b262c0, 0xc000481d40) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:403 +0x4e | |
| created by github.com/weaveworks/common/server.(*Server).Run | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:402 +0x13b | |
| goroutine 800 [select]: | |
| github.com/cortexproject/cortex/pkg/util/runtimeconfig.(*Manager).loop(0xc0006d07e0, 0x2b99c80, 0xc0008c0240, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/util/runtimeconfig/manager.go:132 +0x13f | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0870) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 817 [syscall, 24 minutes]: | |
| os/signal.signal_recv(0x26b85d0) | |
| /usr/local/go/src/runtime/sigqueue.go:147 +0x9c | |
| os/signal.loop() | |
| /usr/local/go/src/os/signal/signal_unix.go:23 +0x22 | |
| created by os/signal.Notify.func1 | |
| /usr/local/go/src/os/signal/signal.go:127 +0x44 | |
| goroutine 596 [select, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*KVInitService).running(0xc000b1fc80, 0x2b99c80, 0xc000bf9980, 0x26b85c8, 0xc0006d05b8) | |
| /__w/cortex/cortex/pkg/ring/kv/memberlist/kv_init_service.go:52 +0xd6 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d05a0) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 813 [semacquire, 5 minutes]: | |
| sync.runtime_Semacquire(0xc1fc5dbfa8) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1fc5dbfa0) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).syncUsersBlocks(0xc0001ce000, 0x2b99c80, 0xc000e9e080, 0x26b38d8, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:225 +0x53e | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).SyncBlocks(0xc0001ce000, 0x2b99c80, 0xc000e9e080, 0x1, 0x2bb6b40) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:138 +0x4b | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).syncStores(0xc000863100, 0x2b99c80, 0xc000e9e080, 0x2538ef7, 0x8) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:312 +0x23e | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).running(0xc000863100, 0x2b99c80, 0xc000e9e080, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:283 +0x47f | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000a0d440) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 814 [chan receive, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc00019c9c0, 0x2ba22c0, 0xc000c0c060) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 815 [chan receive, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc00019ca80, 0x2ba22c0, 0xc000c0c0a0) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 816 [chan receive, 24 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener.func1(0xc00019ccc0, 0x2b7fac0, 0xc000c0c0e0) | |
| /__w/cortex/cortex/pkg/util/services/manager.go:244 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener | |
| /__w/cortex/cortex/pkg/util/services/manager.go:243 +0xfb | |
| goroutine 833 [select]: | |
| github.com/cortexproject/cortex/pkg/ring.(*BasicLifecycler).running(0xc000a0bee0, 0x2b99c80, 0xc000e9e0c0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/ring/basic_lifecycler.go:189 +0x18a | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0ea0) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 834 [select]: | |
| net/http.(*persistConn).roundTrip(0xc27abf85a0, 0xc00d218180, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc000ce5680, 0xc238509200, 0xf8, 0xc000e9d101, 0xc238509200) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc000ce5680, 0xc238509200, 0xc000ce5680, 0xbffdeea00b8fc4bf, 0x1614ea167c3) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc238509100, 0x2b485c0, 0xc000ce5680, 0xbffdeea00b8fc4bf, 0x1614ea167c3, 0x3f6bae0, 0xc00357c648, 0xbffdeea00b8fc4bf, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000bfbe60, 0xc238509100, 0xbffdeea00b8fc4bf, 0x1614ea167c3, 0x3f6bae0, 0xc00357c648, 0x0, 0x1, 0xc2027aa000) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000bfbe60, 0xc238509100, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/hashicorp/consul/api.(*Client).doRequest(0xc000bfc6c0, 0xc13edbcfc0, 0xc000e9d638, 0xc1f5b45800, 0x14, 0xc13edbcfc0) | |
| /__w/cortex/cortex/vendor/github.com/hashicorp/consul/api/api.go:875 +0xb8 | |
| github.com/hashicorp/consul/api.(*KV).getInternal(0xc0001a2a08, 0x2542ee3, 0xd, 0x0, 0xc147d489a0, 0x7f63e741af10, 0xc00122e600, 0x216b600, 0x3f99388) | |
| /__w/cortex/cortex/vendor/github.com/hashicorp/consul/api/kv.go:131 +0x285 | |
| github.com/hashicorp/consul/api.(*KV).Get(0xc0001a2a08, 0x2542ee3, 0xd, 0xc147d489a0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/hashicorp/consul/api/kv.go:65 +0xa0 | |
| github.com/cortexproject/cortex/pkg/ring/kv/consul.consulMetrics.Get.func1(0x2b99d40, 0xc1ea93b530, 0x3, 0xbffdee9b0b8f5a49) | |
| /__w/cortex/cortex/pkg/ring/kv/consul/metrics.go:44 +0x126 | |
| github.com/weaveworks/common/instrument.CollectedRequest(0x2b99d40, 0xc0006780f0, 0x25309b1, 0x3, 0x2b846c0, 0xc000486420, 0x26b5d20, 0xc000e9d980, 0xb0, 0xa0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:152 +0x269 | |
| github.com/cortexproject/cortex/pkg/ring/kv/consul.consulMetrics.Get(0x2ba2480, 0xc0001a2a08, 0x2542ee3, 0xd, 0xc147d489a0, 0xc000e9db60, 0xa8, 0x2adee68, 0xc00007ddd8) | |
| /__w/cortex/cortex/pkg/ring/kv/consul/metrics.go:41 +0x144 | |
| github.com/cortexproject/cortex/pkg/ring/kv/consul.(*Client).WatchKey(0xc0003c8d90, 0x2b99d40, 0xc0006780f0, 0x2542ee3, 0xd, 0xc000cb0090) | |
| /__w/cortex/cortex/pkg/ring/kv/consul/client.go:224 +0x4ff | |
| github.com/cortexproject/cortex/pkg/ring/kv.metrics.WatchKey.func1(0x2b99d40, 0xc0006780f0, 0x8, 0xbffded25f440deda) | |
| /__w/cortex/cortex/pkg/ring/kv/metrics.go:91 +0x5e | |
| github.com/weaveworks/common/instrument.CollectedRequest(0x2b99c80, 0xc000e9e100, 0x25384df, 0x8, 0x2b846c0, 0xc0001a2a20, 0x26b5d20, 0xc000e9ddf0, 0x0, 0x10) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:152 +0x269 | |
| github.com/cortexproject/cortex/pkg/ring/kv.metrics.WatchKey(0x2bb65a0, 0xc0003c8d90, 0xc0001a2a20, 0x2b99c80, 0xc000e9e100, 0x2542ee3, 0xd, 0xc000cb0090) | |
| /__w/cortex/cortex/pkg/ring/kv/metrics.go:90 +0x101 | |
| github.com/cortexproject/cortex/pkg/ring.(*Ring).loop(0xc0006ead80, 0x2b99c80, 0xc000e9e100, 0x26b85c8, 0xc0006d0f48) | |
| /__w/cortex/cortex/pkg/ring/ring.go:235 +0x95 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0f30) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 236067 [semacquire]: | |
| sync.runtime_Semacquire(0xc0182a84c8) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0182a84c0) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc028cbc480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc028cbc480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 884 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc007cd033c, 0xc0f31dbd00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc007cd0338) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc007cd0338) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).unload(0xc007cd02c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:254 +0xd5 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).closeIdleReaders(0xc000b25680) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:109 +0x86 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewReaderPool.func1(0xc000b25680, 0x1bf08eb000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:58 +0x42 | |
| created by github.com/thanos-io/thanos/pkg/block/indexheader.NewReaderPool | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:52 +0x13e | |
| goroutine 4514542 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5886b2e7a8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc06ce7e598, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc06ce7e580, 0xc1fe14c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc06ce7e580, 0xc1fe14c000, 0x8000, 0x8000, 0x20305b, 0x420bb5, 0xc16fdff5a0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc143d460d0, 0xc1fe14c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1cb011f20, 0xc1fafa3458, 0x9, 0x9, 0xc200ac2000, 0x1007f63e740a7d0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1cb011f20, 0xc1fafa3458, 0x9, 0x9, 0x9, 0x601742e6, 0xc079ec9dc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc1fafa3458, 0x9, 0x9, 0x2b41140, 0xc1cb011f20, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc1fafa3420, 0xc265a9a5e0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc054d93b00, 0xc23cc7b020, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc054d93b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc054d93b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4582128 [select]: | |
| net/http.(*persistConn).readLoop(0xc0f78e2240) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4513603 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc175158d70, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc190600300, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc10a44af00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 209257 [semacquire]: | |
| sync.runtime_Semacquire(0xc0b41fb338) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0b41fb330) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0e542a480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0e542a480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4513531 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc078bd3200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582747 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0fc7806c0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4511255 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc047d69360, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d7db40c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1245db680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4514787 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b8c80, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc2442d7bc0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc52a933c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4578948 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0f698b040, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1ea500c00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1e65f3800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 98114 [semacquire]: | |
| sync.runtime_Semacquire(0xc019c06468) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc019c06460) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc00f65e300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc00f65e300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4514703 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59df09f240, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc238281a18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc238281a00, 0xc233518000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc238281a00, 0xc233518000, 0x8000, 0x8000, 0x203066, 0x420bb5, 0xc19b9a1400) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0060fa430, 0xc233518000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc2442d6420, 0xc0460c2f18, 0x9, 0x9, 0x463b40, 0xc0460c2f08, 0xc227d32dc0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc2442d6420, 0xc0460c2f18, 0x9, 0x9, 0x9, 0x56, 0x18, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0460c2f18, 0x9, 0x9, 0x2b41140, 0xc2442d6420, 0x0, 0x0, 0x0, 0xc02d3b8870) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c2ee0, 0xc25836ae50, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc21d477c80, 0xc22f1daff0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc21d477c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc21d477c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4516941 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc003a9e900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4508841 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1ae679630, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d7d860c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc4b81e0600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4517379 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5a90c96ca8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1c30b4398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1c30b4380, 0xc106310000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1c30b4380, 0xc106310000, 0x8000, 0x8000, 0x800010601, 0x0, 0x10601000000e1) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1f017c028, 0xc106310000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1a7997560, 0xc0dd42b378, 0x9, 0x9, 0xc001b1fd40, 0x457243, 0xb5ba6808de07) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1a7997560, 0xc0dd42b378, 0x9, 0x9, 0x9, 0xbffdee9b0a9faceb, 0x15ca5998803, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0dd42b378, 0x9, 0x9, 0x2b41140, 0xc1a7997560, 0x0, 0x0, 0xc255227c20, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0dd42b340, 0xc255227c20, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1038bcf00, 0xc2039253e0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1038bcf00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1038bcf00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4474331 [semacquire]: | |
| sync.runtime_Semacquire(0xc217e2e2b0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc217e2e2a8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc217e2e2a0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1dae62420) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1ed00b7d0, 0x255e040, 0x18, 0xc1ef8073f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1ea4afe60, 0x2bbccc0, 0xc18059e8e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1ea4afe60, 0x7f63bf19f960, 0xc2150fc440, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1ea4afe60, 0x2bbbd60, 0xc2150fc440, 0xc000863100, 0xc0092a3688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc18059e840, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc18059e820, 0xc18059e6c0, 0x26b3908, 0xc18059e820, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc18059e820, 0x2b99d40, 0xc1ed00b680) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18059e800, 0xc18059e6c0, 0xc18059e6e0, 0x7f63e743b5d8, 0xc006081c80) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18059e800, 0xc18059e6c0, 0xc18059e6e0, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18059e800, 0x2b99d40, 0xc1ed00b650) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0xc18059e6c0, 0xc18059e700, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0x1, 0xc18059e680) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0xc18059e6c0, 0xc18059e720, 0x2ac7d009, 0xb59d31f3ee84) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0x203060, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0xc18059e6c0, 0xc18059e740, 0xc18059e6c0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0xc0f099ac68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0xc18059e6c0, 0x26b3908, 0x2b99d40, 0xc1ed00b5f0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876100, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876100, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0182a84c0, 0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876100) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 95630 [semacquire]: | |
| sync.runtime_Semacquire(0xc075a62c58) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc075a62c50) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc071fc8780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc071fc8780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4516942 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59d8f043b8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc238281e18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc238281e00, 0xc142f18000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc238281e00, 0xc142f18000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000072) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0060fa480, 0xc142f18000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1f71bab40, 0xc0a65ff458, 0x9, 0x9, 0xc079ecdd40, 0x457243, 0xb5ba67ff0b46) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1f71bab40, 0xc0a65ff458, 0x9, 0x9, 0x9, 0xbffdee9b0a94b6f1, 0x15ca58e922e, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0a65ff458, 0x9, 0x9, 0x2b41140, 0xc1f71bab40, 0x0, 0x0, 0xc253ccee40, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0a65ff420, 0xc253ccee40, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc003a9e900, 0xc22c875980, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc003a9e900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc003a9e900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4577950 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0fe25c1e0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1ea4a8420, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0c41d3800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4582750 [select]: | |
| net/http.(*persistConn).readLoop(0xc10609c000) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4084494 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc17b38de00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc258f73f20, 0x25, 0x17a4f206, 0x1cfc932, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc258f73f20, 0x25, 0x17a4f206, 0x1cfc932, 0xc1efbc95c0, 0x40cfa8, 0x60, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc258f73f20, 0x25, 0x17a4f206, 0x1cfc932, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc258f73f20, 0x25, 0x17a4f206, 0x1cfc932, 0x0, 0x8000101, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc000c90ae0, 0x2b99c80, 0xc000e9e080, 0xc258f73f20, 0x25, 0x17a4f206, 0x1cfc932, 0x0, 0x7f59e0599060, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000270c00, 0x2b99c80, 0xc000e9e080, 0xc253cca260, 0x20, 0x17a4f206, 0x1cfc932, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc43967fb40, 0x2b44ea0, 0xc19f5801e0, 0xc212664000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xad5fde977f6f7601, 0x5e8f6986b1914cbd, 0xc1bdd68140, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xad5fde977f6f7601, 0x5e8f6986b1914cbd, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xad5fde977f6f7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0034b7b00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4515975 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0de578000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4561337 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc08f604c80, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc21d46e060, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc136f0c900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4566211 [semacquire]: | |
| sync.runtime_Semacquire(0xc1e07b48b0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1e07b48a8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1e07b48a0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1e07b4a50) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc0e785d680, 0x255e040, 0x18, 0xc0fe19d3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1a83f3140, 0x2bbccc0, 0xc0d2c544a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1a83f3140, 0x7f63bf19f960, 0xc1def819d0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1a83f3140, 0x2bbbd60, 0xc1def819d0, 0xc000863100, 0xc0fe19d688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc0d2c54400, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0d2c543e0, 0xc0d2c54280, 0x26b3908, 0xc0d2c543e0, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0d2c543e0, 0x2b99d40, 0xc0e785d530) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0d2c543c0, 0xc0d2c54280, 0xc0d2c542a0, 0x7f63e743ca30, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0d2c543c0, 0xc0d2c54280, 0xc0d2c542a0, 0x2b99d01, 0xc0d2c543c0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0d2c543c0, 0x2b99d40, 0xc0e785d500) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0xc0d2c54280, 0xc0d2c542c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0x1, 0xc0d2c54240) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0xc0d2c54280, 0xc0d2c542e0, 0x2bb9afc2, 0xb5b83509563d) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0x203034, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x40c700, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0xc0d2c54280, 0xc0d2c54300, 0xc0d2c54280, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0xc0005e6c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0xc0d2c54280, 0x26b3908, 0x2b99d40, 0xc0e785d4a0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc1a2f8e780, 0xc278c53e00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc1a2f8e780, 0xc278c53e00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0f7c82930, 0xc0002c3380, 0x2bc6d40, 0xc1a2f8e780, 0xc278c53e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 336244 [semacquire]: | |
| sync.runtime_Semacquire(0xc0051527a8) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0051527a0) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0294cb380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0294cb380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 211498 [semacquire, 1 minutes]: | |
| sync.runtime_Semacquire(0xc0ca383b48) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0ca383b40) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc04ffaa780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc04ffaa780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4514704 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b8a50, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc2442d76e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc21d477e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4582663 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59d97cf9f8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1a0eadf18, 0x72, 0x9800, 0x98a0, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1a0eadf00, 0xc23f932000, 0x98a0, 0x98a0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1a0eadf00, 0xc23f932000, 0x98a0, 0x98a0, 0x203092, 0x420bb5, 0xc2495d8960) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc007b7ef08, 0xc23f932000, 0x98a0, 0x98a0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc2495d8960, 0xc23f932000, 0x98a0, 0x98a0, 0x3448, 0x9893, 0xc01ac02e90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc073195058, 0x2b41380, 0xc2495d8960, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc073194e00, 0x2b48520, 0xc007b7ef08, 0x5, 0xc007b7ef08, 0x3438) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc073194e00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc073194e00, 0xc227dabab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0fa6fe480, 0xc227dabab2, 0x254e, 0x254e, 0x3438, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc23e49a480, 0xc227dabab2, 0x254e, 0x254e, 0x3438, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc1eb15c240, 0xc227dabab2, 0x254e, 0x254e, 0x3438, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc003233300, 0xc227dabab2, 0x254e, 0x254e, 0x3438, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc003233300, 0xc227dabab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc003233340, 0xc227dabab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc003233340, 0xc227da6000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc029bd1380, 0xc029bd13e0, 0xc029bd1440, 0xc00926e860, 0xc225a725d0, 0xc044daa000, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4556672 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f63c00f9be0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc008d7ed18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc008d7ed00, 0xc145b80000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc008d7ed00, 0xc145b80000, 0x8000, 0x8000, 0x20312f, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc005f6a6b8, 0xc145b80000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1ce759500, 0xc07662c038, 0x9, 0x9, 0xc005f19100, 0x1007f63e740a108, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1ce759500, 0xc07662c038, 0x9, 0x9, 0x9, 0x601742e3, 0xc047ef7dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc07662c038, 0x9, 0x9, 0x2b41140, 0xc1ce759500, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc07662c000, 0xc2504f92c0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc183057e00, 0xc2917e8bd0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc183057e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc183057e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4517377 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc08ae15220, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc100420660, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1038bcf00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4582666 [select]: | |
| net/http.(*persistConn).readLoop(0xc0b6675440) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4514796 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b92c0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0c0911800, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1a3430780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4515929 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00bc80d80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4557185 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1563d3130, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1a110a660, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc11159f080) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4486232 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59decac3e0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc22b860118, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc22b860100, 0xc1b5e4a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc22b860100, 0xc1b5e4a000, 0x8000, 0x8000, 0x400000801, 0xc70000000f, 0x8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc004ffb038, 0xc1b5e4a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc12843ae40, 0xc1df964e38, 0x9, 0x9, 0xc1dd153ce0, 0xc13c543df7, 0xc13c543d80) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc12843ae40, 0xc1df964e38, 0x9, 0x9, 0x9, 0x406594, 0xc0b771eae0, 0xc13c543df7) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc1df964e38, 0x9, 0x9, 0x2b41140, 0xc12843ae40, 0x0, 0xc100000000, 0x2124f40, 0xc75045ca70) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc1df964e00, 0xc75045ca60, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1c1c82180, 0xc122530ea0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1c1c82180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1c1c82180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4508536 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0deef0a20) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4568631 [semacquire]: | |
| sync.runtime_Semacquire(0xc0e77e0be0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0e77e0bd8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0e77e0bd0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc0e5e87f80) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc0ffa08ab0, 0x255e040, 0x18, 0xc024c653f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc065e39800, 0x2bbccc0, 0xc1685ff740, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc065e39800, 0x7f63bf19f960, 0xc17d28c520, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc065e39800, 0x2bbbd60, 0xc17d28c520, 0xc000863100, 0xc024c65688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc07f618a80, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc07f618a60, 0xc07f618900, 0x26b3908, 0xc07f618a60, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc07f618a60, 0x2b99d40, 0xc1f17b2000) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc07f618a40, 0xc07f618900, 0xc07f618920, 0x7f63e742ef10, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc07f618a40, 0xc07f618900, 0xc07f618920, 0x2b99d01, 0xc07f618a40) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc07f618a40, 0x2b99d40, 0xc47d55edb0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0xc07f618900, 0xc07f618940, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0x1, 0xc07f6188c0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0xc07f618900, 0xc07f618960, 0x3b244204, 0xb5b84473e87a) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0x20301f, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x40c700, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0xc07f618900, 0xc07f618980, 0xc07f618900, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0xc12b912c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0xc07f618900, 0x26b3908, 0x2b99d40, 0xc47d55ed50) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0de578000, 0xc0ed67af00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0de578000, 0xc0ed67af00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc1296eeaf0, 0xc0002c3380, 0x2bc6d40, 0xc0de578000, 0xc0ed67af00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4582807 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0fd6c8000, 0xc00473c480, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc000ce5540, 0xc0f6b2e200, 0xc11fd686c0, 0x40f967, 0xc0000a7bf8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc000ce5540, 0xc0f6b2e200, 0xc000ce5540, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0f6b2e200, 0x2b485c0, 0xc000ce5540, 0x0, 0x0, 0x0, 0xca27c505f0, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000bfaba0, 0xc0f6b2e200, 0x0, 0x0, 0x0, 0xca27c505f0, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000bfaba0, 0xc0f6b2e200, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc092731bc0, 0xc092731c20, 0xc092731ce0, 0xc1451f5130, 0xc23b892e70, 0xc143aa7d90, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4513606 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc175159e50, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1ba7c7a40, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc10a44bc80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4084492 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc24ae98000, 0xc2270da000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e379920, 0xc2270da000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc23c574b40, 0xc2270da000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0fe80cc80, 0x7f59e0599060, 0xc23c574b40, 0xc2270da000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0fe80cc80, 0x7f59e0599060, 0xc23c574b40, 0xc2270da000, 0x8000, 0x8000, 0x2b6fc80, 0xc23c574b40, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0052ce3c0, 0x2b44ea0, 0xc0fe80cc80, 0xc2270da000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x1cc0a35fbeb97601, 0x161981381d9f8301, 0xc114951b00, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x1cc0a35fbeb97601, 0x161981381d9f8301, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x1cc0a35fbeb97601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0043ebe00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4084804 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc17b38dc80, 0xc20e372000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e379980, 0xc20e372000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc23c574c60, 0xc20e372000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc081243ef0, 0x7f59e0599060, 0xc23c574c60, 0xc20e372000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc081243ef0, 0x7f59e0599060, 0xc23c574c60, 0xc20e372000, 0x8000, 0x8000, 0x2b6fc80, 0xc23c574c60, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0026ad940, 0x2b44ea0, 0xc081243ef0, 0xc20e372000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x2856316f9d2d7701, 0xcee3ec3285783cc7, 0xc143f83080, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x2856316f9d2d7701, 0xcee3ec3285783cc7, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x2856316f9d2d7701, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc007568400, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4581870 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc007cd0344, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc007cd02c0, 0xc0d278c2c8, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294 | |
| github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc1564447d0, 0xc114311bf0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1693 +0x6f0 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc99c97fc00, 0xc07df928b0, 0x2, 0x2, 0x2d850e0a0cd5be5, 0x7854ca83079dac14, 0x61032ff49c49aef4, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7 | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0040c6990, 0xc99c97fc00, 0xc121ce5d40, 0xc07df928b0, 0x2, 0x2, 0xc121ce5920, 0x2b44f80, 0xc114311cb0, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc21d477500, 0xc1297e9800) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc114311c80, 0xc150d53f10) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4582835 [select]: | |
| net/http.(*persistConn).writeLoop(0xc10297e000) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4574119 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00d68dc04, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00d68db80, 0xc03f1468b0, 0xb, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294 | |
| github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc1e89f57d0, 0xc1e07b4780, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1713 +0x403 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc9a77a0f80, 0xc181398a80, 0x3, 0x3, 0xc255ef5390, 0xd, 0x1, 0xc1ed1fca80, 0x7f63e7423ca0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7 | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0077aba70, 0xc9a77a0f80, 0xc1d7d80c60, 0xc181398a80, 0x3, 0x3, 0xc1a83f3140, 0x2b44f80, 0xc1e07b48d0, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x1, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1e07b48a0, 0xc00915c690) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4582748 [select]: | |
| net/http.(*persistConn).readLoop(0xc0d0a00d80) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4513292 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc2a2000190, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1e6948060, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc065466f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 347184 [semacquire, 3 minutes]: | |
| sync.runtime_Semacquire(0xc0eb34a3d8) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0eb34a3d0) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc09673ec00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc09673ec00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 97730 [semacquire]: | |
| sync.runtime_Semacquire(0xc02c9b5d28) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc02c9b5d20) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0073fe780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0073fe780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4514541 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc054d93b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4562284 [semacquire]: | |
| sync.runtime_Semacquire(0xc1ea93a460) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1ea93a458) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1ea93a450, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1ea93a9c0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1ea93a390, 0x255e040, 0x18, 0xc249d033f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc13edbc1e0, 0x2bbccc0, 0xc1ea9303c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc13edbc1e0, 0x7f63bf19f960, 0xc000aa40f0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc13edbc1e0, 0x2bbbd60, 0xc000aa40f0, 0xc000863100, 0xc101347688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1ea930320, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1ea930300, 0xc1ea9301a0, 0x26b3908, 0xc1ea930300, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1ea930300, 0x2b99d40, 0xc1ea93a240) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea9302e0, 0xc1ea9301a0, 0xc1ea9301c0, 0x7f63e741af10, 0xc06fbf1380) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea9302e0, 0xc1ea9301a0, 0xc1ea9301c0, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea9302e0, 0x2b99d40, 0xc1ea93a210) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0xc1ea9301a0, 0xc1ea9301e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0x1, 0xc1ea930160) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0xc1ea9301a0, 0xc1ea930200, 0xa94b9e9, 0xb5ba67f04466) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0x20307a, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0xc1ea9301a0, 0xc1ea930220, 0xc1ea9301a0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0xc05d8b0c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0xc1ea9301a0, 0x26b3908, 0x2b99d40, 0xc1ea93a1b0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc238508a00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc238508a00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc12c6a8860, 0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc238508a00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4577952 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59d9b17ad0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc181fd1518, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc181fd1500, 0xc1ad662000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc181fd1500, 0xc1ad662000, 0x8000, 0x8000, 0x10401, 0x0, 0x1040102ade864) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc150ab49b8, 0xc1ad662000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc0bf00a360, 0xc0f27018b8, 0x9, 0x9, 0xc0061ee700, 0x7f63e742e180, 0xc06fbf1c80) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc0bf00a360, 0xc0f27018b8, 0x9, 0x9, 0x9, 0xa6fc25, 0xc110468cf0, 0xc110460004) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0f27018b8, 0x9, 0x9, 0x2b41140, 0xc0bf00a360, 0x0, 0x0, 0xc110468cf0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f2701880, 0xc110468cf0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0c41d3800, 0xc1136af7a0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0c41d3800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0c41d3800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4582833 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f78e2240) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4532623 [semacquire]: | |
| sync.runtime_Semacquire(0xc27a2de2e0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc27a2de2d8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc27a2de2d0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc27477a960) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc12dce4420, 0x255e040, 0x18, 0xc2394e93f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc13de3c660, 0x2bbccc0, 0xc06e14d200, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc13de3c660, 0x7f63bf19f960, 0xc1fa720ac0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc13de3c660, 0x2bbbd60, 0xc1fa720ac0, 0xc000863100, 0xc2394e9688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc06e14d160, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc06e14d140, 0xc06e14cfe0, 0x26b3908, 0xc06e14d140, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc06e14d140, 0x2b99d40, 0xc12dce42d0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc06e14d120, 0xc06e14cfe0, 0xc06e14d000, 0x7f63e741b5d8, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc06e14d120, 0xc06e14cfe0, 0xc06e14d000, 0x2b99d01, 0xc06e14d120) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc06e14d120, 0x2b99d40, 0xc12dce42a0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0xc06e14cfe0, 0xc06e14d020, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0x1, 0xc06e14cfa0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0xc06e14cfe0, 0xc06e14d040, 0xf1194f4, 0xb5b5c455576a) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0x20301b, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x7f59e185f400, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0xc06e14cfe0, 0xc06e14d060, 0xc06e14cfe0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0xc157146c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0xc06e14cfe0, 0x26b3908, 0x2b99d40, 0xc12dce4240) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc12dde6b00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc12dde6b00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc126847560, 0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc12dde6b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4582523 [select]: | |
| net/http.(*persistConn).writeLoop(0xc9aa80f7a0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4565087 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1ecc2c480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4557187 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f63bf05f728, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc06ce7e318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc06ce7e300, 0xc1de1bc000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc06ce7e300, 0xc1de1bc000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000006) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc143d460a0, 0xc1de1bc000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc14ff80600, 0xc110dfcd58, 0x9, 0x9, 0xc071596d40, 0x457243, 0xb5b93b52c6bc) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc14ff80600, 0xc110dfcd58, 0x9, 0x9, 0x9, 0xbffdee99c7eec8e9, 0x15b78e2b1ff, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc110dfcd58, 0x9, 0x9, 0x2b41140, 0xc14ff80600, 0x0, 0x0, 0xc25c7680c0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc110dfcd20, 0xc25c7680c0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc11159f080, 0xc27bf06270, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc11159f080) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc11159f080) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4513608 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59d8f04738, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00cfa2718, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00cfa2700, 0xc1ffa5e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc00cfa2700, 0xc1ffa5e000, 0x8000, 0x8000, 0x20308e, 0x420bb5, 0xc23a5d9d80) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc9e49f40f8, 0xc1ffa5e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc10cfe4fc0, 0xc1b5e6c658, 0x9, 0x9, 0xc00596d180, 0x1007f63e742a848, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc10cfe4fc0, 0xc1b5e6c658, 0x9, 0x9, 0x9, 0x601742dd, 0xc003bfddc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc1b5e6c658, 0x9, 0x9, 0x2b41140, 0xc10cfe4fc0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc1b5e6c620, 0xc0168a5f80, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc10a44bc80, 0xc24500a720, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc10a44bc80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc10a44bc80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4511072 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f58819b36b0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc16897a498, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc16897a480, 0xc249901000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc16897a480, 0xc249901000, 0x1000, 0x1000, 0x4db33a, 0xb5bb70dfc9e3, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc008464108, 0xc249901000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| net/http.(*connReader).Read(0xc2080077a0, 0xc249901000, 0x1000, 0x1000, 0xbffdeeb3102f39c8, 0x1730534d4cc, 0x3f6bae0) | |
| /usr/local/go/src/net/http/server.go:797 +0xf4 | |
| bufio.(*Reader).fill(0xc1ab3ef920) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1ab3ef920, 0x4, 0x1730534d4cc, 0x3f6bae0, 0x0, 0x0, 0x3f6bae0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*conn).serve(0xc245df2140, 0x2b99c80, 0xc1ebb5eac0) | |
| /usr/local/go/src/net/http/server.go:1949 +0x9d7 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:2962 +0x35c | |
| goroutine 4514647 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59c9f7d390, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1c30b4218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1c30b4200, 0xc22f1dc000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1c30b4200, 0xc22f1dc000, 0x8000, 0x8000, 0x400000801, 0x1, 0x8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1f017c010, 0xc22f1dc000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc23b73ac60, 0xc0460c2c78, 0x9, 0x9, 0xc22f1dc034, 0x4, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc23b73ac60, 0xc0460c2c78, 0x9, 0x9, 0x9, 0xc071597dc8, 0x40cfa8, 0x20) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0460c2c78, 0x9, 0x9, 0x2b41140, 0xc23b73ac60, 0x0, 0xc100000000, 0x2124f40, 0xc156f1e048) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c2c40, 0xc156f1e050, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc21d477200, 0xc22f1da510, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc21d477200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc21d477200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4464565 [running]: | |
| goroutine running on other thread; stack unavailable | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4515976 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5a90c95dc8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc23a27d418, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc23a27d400, 0xc13642c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc23a27d400, 0xc13642c000, 0x8000, 0x8000, 0x800010601, 0x0, 0x106010000001a) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc52a84a000, 0xc13642c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1d9dfc2a0, 0xc244dee1f8, 0x9, 0x9, 0xc0220bad40, 0x457243, 0xb5b93c5069f8) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1d9dfc2a0, 0xc244dee1f8, 0x9, 0x9, 0x9, 0xbffdee99c8e997ee, 0x15b79dd8106, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc244dee1f8, 0x9, 0x9, 0x2b41140, 0xc1d9dfc2a0, 0x0, 0x0, 0xc260132340, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc244dee1c0, 0xc260132340, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0de578000, 0xc241efc4b0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0de578000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0de578000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 235963 [semacquire, 3 minutes]: | |
| sync.runtime_Semacquire(0xc03f97d978) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc03f97d970) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc023e1ac00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc023e1ac00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 347451 [semacquire, 3 minutes]: | |
| sync.runtime_Semacquire(0xc0b0cfd638) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0b0cfd630) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0158e2480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0158e2480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4510185 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0cae88cd0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1e950a960, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc19966bb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4566154 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0d9ceeb40, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1acbcfd40, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0ab132a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4582746 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59d9810148, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1e8c28498, 0x72, 0xe100, 0xe1c1, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1e8c28480, 0xc25b8e0000, 0xe1c1, 0xe1c1, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1e8c28480, 0xc25b8e0000, 0xe1c1, 0xe1c1, 0x203092, 0x420bb5, 0xc24bf39da0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc029070e18, 0xc25b8e0000, 0xe1c1, 0xe1c1, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc24bf39da0, 0xc25b8e0000, 0xe1c1, 0xe1c1, 0x0, 0x41310b, 0xc22856c900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc2011ad058, 0x2b41380, 0xc24bf39da0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc2011ace00, 0x2b48520, 0xc029070e18, 0x5, 0xc029070e18, 0x40c924) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc2011ace00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc2011ace00, 0xc23dd31000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0fc7806c0, 0xc23dd31000, 0x1000, 0x1000, 0xc072dc6300, 0xc22856cc58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc23e84e180) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc23e84e180, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0ca347500) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0fc7806c0) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4511300 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59deb9dc60, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1a5d31d98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1a5d31d80, 0xc1ff776000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1a5d31d80, 0xc1ff776000, 0x8000, 0x8000, 0x400000801, 0xc100000001, 0x8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc150cbc078, 0xc1ff776000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1d9d0fb60, 0xc0c3c41538, 0x9, 0x9, 0xc1ff776027, 0x4, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1d9d0fb60, 0xc0c3c41538, 0x9, 0x9, 0x9, 0xc00169ddc8, 0x40cfa8, 0x20) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0c3c41538, 0x9, 0x9, 0x2b41140, 0xc1d9d0fb60, 0x0, 0xc100000000, 0x2124f40, 0xc105e7fad8) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0c3c41500, 0xc105e7faf0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc167bf0900, 0xc1d77a48d0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc167bf0900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc167bf0900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4501209 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00c4186b0, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).Lock(0xc00c4186a8) | |
| /usr/local/go/src/sync/rwmutex.go:103 +0x88 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc00c418630, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c418630, 0xc1858568e1, 0x3, 0x7f4c11e8dc72, 0xe, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1ea70f500, 0xc06b51aa00, 0x6, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1ea70f500, 0xc1a577d020, 0x2, 0x2, 0xc07905b480, 0xd, 0x1, 0xc005e16e00, 0x7f63e7424a30) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc004d1b1a0, 0xc1ea70f500, 0xc1d9cb25a0, 0xc1a577d020, 0x2, 0x2, 0xc182805b00, 0x2b44f80, 0xc245857890, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc17c65c740, 0xc236e088b0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc245857860, 0xc14393b960) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4522981 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5887153e30, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc238281d98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc238281d80, 0xc107cba000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc238281d80, 0xc107cba000, 0x8000, 0x8000, 0x203060, 0x420bb5, 0xc180257760) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0060fa478, 0xc107cba000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc198be6de0, 0xc1c4a3ee38, 0x9, 0x9, 0xc005ea3180, 0x1007f63e741ca30, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc198be6de0, 0xc1c4a3ee38, 0x9, 0x9, 0x9, 0x601742e6, 0xc079ec8dc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc1c4a3ee38, 0x9, 0x9, 0x2b41140, 0xc198be6de0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc1c4a3ee00, 0xc25ecfd460, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc4f1083800, 0xc00bde8cc0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc4f1083800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc4f1083800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4514788 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc52a933c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4514916 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1a3430f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4565086 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0cdc1e780, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1359838c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1ecc2c480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4581469 [semacquire]: | |
| sync.runtime_Semacquire(0xc1bc0bd7b0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1bc0bd7a8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1bc0bd7a0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1ae047bf0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1bc0bd6b0, 0x255e040, 0x18, 0xc15f1ab3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1893ba060, 0x2bbccc0, 0xc1e902a400, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1893ba060, 0x7f63bf19f960, 0xc00c028430, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1893ba060, 0x2bbbd60, 0xc00c028430, 0xc000863100, 0xc0ff3c1688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1e902a360, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e902a340, 0xc1e902a1e0, 0x26b3908, 0xc1e902a340, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e902a340, 0x2b99d40, 0xc1bc0bd560) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e902a320, 0xc1e902a1e0, 0xc1e902a200, 0x7f63e741fca0, 0xc0b6aae000) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e902a320, 0xc1e902a1e0, 0xc1e902a200, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e902a320, 0x2b99d40, 0xc1bc0bd530) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0xc1e902a1e0, 0xc1e902a220, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0x1, 0xc1e902a1a0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0xc1e902a1e0, 0xc1e902a240, 0x15d5fd4b, 0xb5b9c06129c3) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0x20307a, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0xc1e902a1e0, 0xc1e902a260, 0xc1e902a1e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0xc0005eac68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0xc1e902a1e0, 0x26b3908, 0x2b99d40, 0xc1bc0bd4d0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc0f6f21b00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc0f6f21b00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc12c6a8860, 0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc0f6f21b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4514800 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1503a02d0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0c0911b60, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1a3430c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4516666 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0b393b4a0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc14c6d0d80, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc2fd38d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4514700 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5885067988, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc06ce7e298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc06ce7e280, 0xc233500000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc06ce7e280, 0xc233500000, 0x8000, 0x8000, 0x20308b, 0x420bb5, 0xc22c886400) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc143d46098, 0xc233500000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc23b73bf80, 0xc0460c2e38, 0x9, 0x9, 0xc006200700, 0x1007f63e7426180, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc23b73bf80, 0xc0460c2e38, 0x9, 0x9, 0x9, 0x601742dd, 0xc0594ebdc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0460c2e38, 0x9, 0x9, 0x2b41140, 0xc23b73bf80, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c2e00, 0xc16d8efe40, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc21d477b00, 0xc22f1dade0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc21d477b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc21d477b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4514794 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1a3430180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4515946 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59c9d0d468, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1c30b4498, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1c30b4480, 0xc2023f2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1c30b4480, 0xc2023f2000, 0x8000, 0x8000, 0x203039, 0x420bb5, 0xc0e721c2a0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1f017c038, 0xc2023f2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc10cfe4f60, 0xc04a34eb98, 0x9, 0x9, 0xc1d4e89c00, 0x1007f63e74290f8, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc10cfe4f60, 0xc04a34eb98, 0x9, 0x9, 0x9, 0x601742e6, 0xc038e1cdc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc04a34eb98, 0x9, 0x9, 0x2b41140, 0xc10cfe4f60, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc04a34eb60, 0xc26727f5e0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc028eec180, 0xc2459a0630, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc028eec180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc028eec180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4514791 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc52a933e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4513607 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc10a44bc80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582153 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00ddea134, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).IndexVersion(0xc00ddea0b0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:141 +0x1f7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0049f9300, 0xc084f17b90, 0x2, 0x2, 0x55d6ca13f4bfe0e2, 0x23aeb4e8816c5b14, 0x6a35026af49e6978, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1639 +0xfbf | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00618cb10, 0xc0049f9300, 0xc14a10a120, 0xc084f17b90, 0x2, 0x2, 0xc197b33380, 0x2b44f80, 0xc1e8866780, 0xc00017c0b8, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1e8866750, 0xc1e8b404d0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4515935 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59decad100, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1528d3698, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1528d3680, 0xc2473f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1528d3680, 0xc2473f4000, 0x8000, 0x8000, 0x203059, 0x420bb5, 0xc167ccaea0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1cb614160, 0xc2473f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc19149f860, 0xc1df964578, 0x9, 0x9, 0xc00d1b5138, 0xc000894d60, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc19149f860, 0xc1df964578, 0x9, 0x9, 0x9, 0xc000894dc8, 0x40cfa8, 0xc07661ae70) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc1df964578, 0x9, 0x9, 0x2b41140, 0xc19149f860, 0x0, 0x0, 0x0, 0xc07661ae60) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc1df964540, 0xc1f3148c80, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00bc80f00, 0xc23ec67a10, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc00bc80f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc00bc80f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4516667 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc2fd38d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582737 [runnable]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0ca346720, 0xc0ca346780, 0xc0ca346840, 0xc00d118950, 0xc1e8b46ea0, 0xc142ad7a20, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4582274 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f599ab310a0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc19c7d8b98, 0x72, 0xdd00, 0xddd6, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc19c7d8b80, 0xc241cbe000, 0xddd6, 0xddd6, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc19c7d8b80, 0xc241cbe000, 0xddd6, 0xddd6, 0x203094, 0x420bb5, 0xc253c642e0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc150304b98, 0xc241cbe000, 0xddd6, 0xddd6, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc253c642e0, 0xc241cbe000, 0xddd6, 0xddd6, 0x1200, 0xddc9, 0xc0aad98e90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc07ab77058, 0x2b41380, 0xc253c642e0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc07ab76e00, 0x2b48520, 0xc150304b98, 0x5, 0xc150304b98, 0x11f0) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc07ab76e00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc07ab76e00, 0xc224287e42, 0x61be, 0x61be, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc24422a7e0, 0xc224287e42, 0x61be, 0x61be, 0x11f0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc23e379680, 0xc224287e42, 0x61be, 0x61be, 0x11f0, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc23b13d340, 0xc224287e42, 0x61be, 0x61be, 0x11f0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc00702a680, 0xc224287e42, 0x61be, 0x61be, 0x11f0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc00702a680, 0xc224287e42, 0x61be, 0x61be, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc00702a6c0, 0xc224287e42, 0x61be, 0x61be, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc00702a6c0, 0xc224286000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc26a56a240, 0xc26a56a2a0, 0xc26a56a300, 0xc000aa4e50, 0xc1ea93b710, 0xc147d48b00, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4577212 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc52a6fd200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4514645 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc022deff40, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc23b73b6e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc21d477200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4510177 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1a79dd180, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc22a263da0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc19966b800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4388238 [running]: | |
| goroutine running on other thread; stack unavailable | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4514691 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc21d477500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4502851 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c51d1e0, 0xc14319a860, 0x8, 0xc1507c8780, 0x28, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1e6c36a00, 0xc0f27e6c40, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1e6c36a00, 0xc1c746a818, 0x1, 0x1, 0x28bcc65066b49d01, 0x30077b56078f8525, 0x3e90ff60c125ea02, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc1e6c36a00, 0xc14a332c00, 0xc1c746a818, 0x1, 0x1, 0xc088e9d200, 0x2b44f80, 0xc036d84ff0, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc165568ee0, 0xc17c325dd0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc036d84fc0, 0xc00d5bed20) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4464215 [running]: | |
| goroutine running on other thread; stack unavailable | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4582839 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f7ded7a0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4514792 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59c9c50a88, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc238281b18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc238281b00, 0xc233560000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc238281b00, 0xc233560000, 0x8000, 0x8000, 0x4647d0, 0xc1734a2300, 0x4) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0060fa450, 0xc233560000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc2442d79e0, 0xc0460c31b8, 0x9, 0x9, 0xc22f1f51f8, 0x461d40, 0xc22f1f51a0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc2442d79e0, 0xc0460c31b8, 0x9, 0x9, 0x9, 0xc0594e5d01, 0x10000000040cfa8, 0xc02d3b8e70) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0460c31b8, 0x9, 0x9, 0x2b41140, 0xc2442d79e0, 0x0, 0x0, 0x0, 0xc02d3b8e60) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c3180, 0xc1a9c36770, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc52a933e00, 0xc22f1db620, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc52a933e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc52a933e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4582542 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5887154450, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc23b12a018, 0x72, 0xe500, 0xe506, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc23b12a000, 0xc25b8b4000, 0xe506, 0xe506, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc23b12a000, 0xc25b8b4000, 0xe506, 0xe506, 0x203092, 0x420bb5, 0xc2494fa9a0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc150304b90, 0xc25b8b4000, 0xe506, 0xe506, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc2494fa9a0, 0xc25b8b4000, 0xe506, 0xe506, 0x1f38, 0xe4f9, 0xc1afb0ee90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc07ab76cd8, 0x2b41380, 0xc2494fa9a0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc07ab76a80, 0x2b48520, 0xc150304b90, 0x5, 0xc150304b90, 0x1f28) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc07ab76a80, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc07ab76a80, 0xc2110fdab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0f78e2240, 0xc2110fdab2, 0x254e, 0x254e, 0x1f28, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc135863e60, 0xc2110fdab2, 0x254e, 0x254e, 0x1f28, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc23b13d720, 0xc2110fdab2, 0x254e, 0x254e, 0x1f28, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc00702a700, 0xc2110fdab2, 0x254e, 0x254e, 0x1f28, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc00702a700, 0xc2110fdab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc00702a780, 0xc2110fdab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc00702a780, 0xc2110f8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc052bfca20, 0xc052bfca80, 0xc052bfcae0, 0xc00d5805c0, 0xc20cd8d7d0, 0xc143aa7810, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4582837 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0fb8f86c0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4084802 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc06549cd80, 0xc223fa8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc147ddba40, 0xc223fa8000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc2079cb800, 0xc223fa8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc112f10050, 0x7f59e0599060, 0xc2079cb800, 0xc223fa8000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc112f10050, 0x7f59e0599060, 0xc2079cb800, 0xc223fa8000, 0x8000, 0x8000, 0x2b6fc80, 0xc2079cb800, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0052ce5c0, 0x2b44ea0, 0xc112f10050, 0xc223fa8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xafc070deee727601, 0x602716a9a5bef5f3, 0xc1bdd69600, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xafc070deee727601, 0x602716a9a5bef5f3, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xafc070deee727601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc001ae1e00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4501842 [semacquire]: | |
| sync.runtime_Semacquire(0xc036d84fd0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc036d84fc8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc036d84fc0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1e5f75050) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc036d84f30, 0x255e040, 0x18, 0xc130c313f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc088e9d200, 0x2bbccc0, 0xc03785a900, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc088e9d200, 0x7f63bf19f960, 0xc0026ce730, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc088e9d200, 0x2bbbd60, 0xc0026ce730, 0xc000863100, 0xc240db7688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc03785a860, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc03785a840, 0xc03785a6e0, 0x26b3908, 0xc03785a840, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc03785a840, 0x2b99d40, 0xc036d84300) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc03785a820, 0xc03785a6e0, 0xc03785a700, 0x7f63e742f5d8, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc03785a820, 0xc03785a6e0, 0xc03785a700, 0x2b99d01, 0xc03785a820) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc03785a820, 0x2b99d40, 0xc036d842d0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0xc03785a6e0, 0xc03785a720, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0x1, 0xc03785a6a0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0xc03785a6e0, 0xc03785a740, 0xdf79c1, 0xb5b1fc769c44) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0x20300d, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x7f59d9bbc400, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0xc03785a6e0, 0xc03785a760, 0xc03785a6e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0xc02ad75c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0xc03785a6e0, 0x26b3908, 0x2b99d40, 0xc036d84270) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1664d0200, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1664d0200, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0b41fb330, 0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1664d0200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4555621 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc081fa6900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582080 [select]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc01ade3620, 0xc01ade3680, 0xc01ade36e0, 0xc009911e70, 0xc1e8866cc0, 0xc1068bf970, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4084487 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc2cfa40900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc2570e5170, 0x25, 0x2886d48d, 0x26f0620, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc2570e5170, 0x25, 0x2886d48d, 0x26f0620, 0xc1273fb5c0, 0x40cfa8, 0x60, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc2570e5170, 0x25, 0x2886d48d, 0x26f0620, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc2570e5170, 0x25, 0x2886d48d, 0x26f0620, 0x0, 0x2000107, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc000c90ae0, 0x2b99c80, 0xc000e9e080, 0xc2570e5170, 0x25, 0x2886d48d, 0x26f0620, 0x0, 0x7f59e0599060, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000270c00, 0x2b99c80, 0xc000e9e080, 0xc1f6a00220, 0x20, 0x2886d48d, 0x26f0620, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc0052ce1c0, 0x2b44ea0, 0xc107d13090, 0xc21fca6000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xfae3d9fe9f627601, 0xb0169f34b529a016, 0xc115d56100, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xfae3d9fe9f627601, 0xb0169f34b529a016, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xfae3d9fe9f627601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc002599200, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4514702 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc21d477c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582744 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f59c9c50468, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc20ac7f318, 0x72, 0xd400, 0xd401, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc20ac7f300, 0xc2492e0000, 0xd401, 0xd401, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc20ac7f300, 0xc2492e0000, 0xd401, 0xd401, 0x203094, 0x420bb5, 0xc25387a2a0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc007b7ef10, 0xc2492e0000, 0xd401, 0xd401, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc25387a2a0, 0xc2492e0000, 0xd401, 0xd401, 0x0, 0x41310b, 0xc02bbd9900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0731953d8, 0x2b41380, 0xc25387a2a0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc073195180, 0x2b48520, 0xc007b7ef10, 0x5, 0xc007b7ef10, 0x40c924) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc073195180, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc073195180, 0xc23dd2f000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0f84707e0, 0xc23dd2f000, 0x1000, 0x1000, 0xc072dc6060, 0xc02bbd9c58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc23e84e000) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc23e84e000, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0ca347440) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0f84707e0) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4517378 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1038bcf00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582303 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0fa6fe480) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4084805 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1f0dbad80, 0xc227da6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc11af77020, 0xc227da6000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc1ea0b58f0, 0xc227da6000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0fa984280, 0x7f59e0599060, 0xc1ea0b58f0, 0xc227da6000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0fa984280, 0x7f59e0599060, 0xc1ea0b58f0, 0xc227da6000, 0x8000, 0x8000, 0x2b6fc80, 0xc1ea0b58f0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc14ee7f5c0, 0x2b44ea0, 0xc0fa984280, 0xc227da6000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xff02bb445c447701, 0x1e671c2618fcb70b, 0xc144281300, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xff02bb445c447701, 0x1e671c2618fcb70b, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xff02bb445c447701, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc007f18000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4581515 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0f84707e0, 0xc004d64340, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc000ce5540, 0xc12e007600, 0xc1f071a6c0, 0x40f967, 0xc0000874c8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc000ce5540, 0xc12e007600, 0xc000ce5540, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc12e007600, 0x2b485c0, 0xc000ce5540, 0x0, 0x0, 0x0, 0xc1c746b458, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000bfaba0, 0xc12e007600, 0x0, 0x0, 0x0, 0xc1c746b458, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000bfaba0, 0xc12e007600, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1863eaf60, 0xc1863eafc0, 0xc1863eb020, 0xc445ab91a0, 0xc192611710, 0xc109318e70, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4510179 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59dbd66458, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1ebaee118, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1ebaee100, 0xc0c61c4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1ebaee100, 0xc0c61c4000, 0x8000, 0x8000, 0x203087, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc004ffa028, 0xc0c61c4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc22a2638c0, 0xc12d6663b8, 0x9, 0x9, 0xc001c91500, 0x1007f63e7434368, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc22a2638c0, 0xc12d6663b8, 0x9, 0x9, 0x9, 0x601742e3, 0xc22c8bedc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc12d6663b8, 0x9, 0x9, 0x2b41140, 0xc22a2638c0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc12d666380, 0xc252fc4f20, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc19966b800, 0xc23e2b1a40, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc19966b800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc19966b800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4582749 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0d0a00d80) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4514698 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b8690, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc2442d64e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc21d477b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4513294 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59d9b65f18, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc06ce7e418, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc06ce7e400, 0xc14875c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc06ce7e400, 0xc14875c000, 0x8000, 0x8000, 0x800010601, 0x0, 0x106010000008f) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc143d460b0, 0xc14875c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc0ff161e60, 0xc00507c038, 0x9, 0x9, 0xc038e1ad40, 0x457243, 0xb5b93c2a95d0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc0ff161e60, 0xc00507c038, 0x9, 0x9, 0x9, 0xbffdee99c8cb24a2, 0x15b79bf0db2, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc00507c038, 0x9, 0x9, 0x2b41140, 0xc0ff161e60, 0x0, 0x0, 0xc25e9bd1e0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00507c000, 0xc25e9bd1e0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc065466f00, 0xc24859a210, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc065466f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc065466f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4515933 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc07661ae60, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19149f980, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00bc80f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4582131 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00ddea134, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00ddea0b0, 0xc1580704d0, 0x8, 0xc25a00bc50, 0x21, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc197c69480, 0xc1ea9307a0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc197c69480, 0xc00357c4c8, 0x1, 0x1, 0xc0d39ff800, 0xd, 0x1, 0xc001c16380, 0x7f63e741a848) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00618cb10, 0xc197c69480, 0xc13edbc7e0, 0xc00357c4c8, 0x1, 0x1, 0xc13edbc1e0, 0x2b44f80, 0xc1ea93a480, 0xc00017c0b8, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1ea93a450, 0xc1e8aaf570) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4464265 [semacquire, 3 minutes]: | |
| sync.runtime_Semacquire(0xc238bf2be0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc238bf2bd8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc238bf2bd0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc239202960) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc238bf2b10, 0x255e040, 0x18, 0xc1ea4f13f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc238bf4600, 0x2bbccc0, 0xc067f04580, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc238bf4600, 0x7f63bf19f960, 0xc238bfc030, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc238bf4600, 0x2bbbd60, 0xc238bfc030, 0xc000863100, 0xc238e17688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc067f044c0, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc067f04480, 0xc067f042a0, 0x26b3908, 0xc067f04480, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc067f04480, 0x2b99d40, 0xc238bf29c0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc067f04460, 0xc067f042a0, 0xc067f042e0, 0x7f63e743a180, 0xc0e13cfe00) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc067f04460, 0xc067f042a0, 0xc067f042e0, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc067f04460, 0x2b99d40, 0xc238bf2990) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0xc067f042a0, 0xc067f04300, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0x1, 0xc067f04260) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0xc067f042a0, 0xc067f04320, 0x293891ad, 0xb57b321d7c2c) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0x203019, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0xc067f042a0, 0xc067f04360, 0xc067f042a0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0xc0f0ed7c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0xc067f042a0, 0x26b3908, 0x2b99d40, 0xc238bf2930) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0073fe780, 0xc238bbeb00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0073fe780, 0xc238bbeb00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc02c9b5d20, 0xc0002c3380, 0x2bc6d40, 0xc0073fe780, 0xc238bbeb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4582327 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f59deb9d640, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1fffd3398, 0x72, 0xdd00, 0xddd6, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1fffd3380, 0xc249d50000, 0xddd6, 0xddd6, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1fffd3380, 0xc249d50000, 0xddd6, 0xddd6, 0x203094, 0x420bb5, 0xc25273e740) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc150ab4cf8, 0xc249d50000, 0xddd6, 0xddd6, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc25273e740, 0xc249d50000, 0xddd6, 0xddd6, 0x3668, 0x5555, 0xc0fffcae90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc07a507758, 0x2b41380, 0xc25273e740, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc07a507500, 0x2b48520, 0xc150ab4cf8, 0x5, 0xc150ab4cf8, 0x3658) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc07a507500, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc07a507500, 0xc223faebfa, 0x1406, 0x1406, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0d0a00d80, 0xc223faebfa, 0x1406, 0x1406, 0x3658, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc23e84e300, 0xc223faebfa, 0x1406, 0x1406, 0x3658, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc22509f560, 0xc223faebfa, 0x1406, 0x1406, 0x3658, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc1a42198c0, 0xc223faebfa, 0x1406, 0x1406, 0x3658, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc1a42198c0, 0xc223faebfa, 0x1406, 0x1406, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc1a4219900, 0xc223faebfa, 0x1406, 0x1406, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc1a4219900, 0xc223fa8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc16aa90ae0, 0xc16aa90b40, 0xc16aa90ba0, 0xc07df935c0, 0xc1ea0b5590, 0xc0a7228bb0, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4581869 [semacquire]: | |
| sync.runtime_Semacquire(0xc114311c90) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc114311c88) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc114311c80, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1b272fc80) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc114311b60, 0x255e040, 0x18, 0xc0f55573f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc121ce5920, 0x2bbccc0, 0xc1ea0a9d80, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc121ce5920, 0x7f63bf19f960, 0xc07df92710, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc121ce5920, 0x2bbbd60, 0xc07df92710, 0xc000863100, 0xc23f485688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1ea0a9ce0, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1ea0a9cc0, 0xc1ea0a9b60, 0x26b3908, 0xc1ea0a9cc0, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1ea0a9cc0, 0x2b99d40, 0xc1143119b0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea0a9ca0, 0xc1ea0a9b60, 0xc1ea0a9b80, 0x7f63e742d0f8, 0xc00e0ad680) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea0a9ca0, 0xc1ea0a9b60, 0xc1ea0a9b80, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea0a9ca0, 0x2b99d40, 0xc114311980) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0xc1ea0a9b60, 0xc1ea0a9ba0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0x1, 0xc1ea0a9b20) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0xc1ea0a9b60, 0xc1ea0a9bc0, 0x2df8cb00, 0xb5b9d883f77b) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0x20307a, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0xc1ea0a9b60, 0xc1ea0a9be0, 0xc1ea0a9b60, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0xc00211ac68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0xc1ea0a9b60, 0x26b3908, 0x2b99d40, 0xc114311920) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc21d477500, 0xc1297e9800, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc21d477500, 0xc1297e9800, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc06d5bfa20, 0xc0002c3380, 0x2bc6d40, 0xc21d477500, 0xc1297e9800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4582173 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f5886ec6b98, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc19e32ff18, 0x72, 0xe300, 0xe36e, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc19e32ff00, 0xc25b8d0000, 0xe36e, 0xe36e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc19e32ff00, 0xc25b8d0000, 0xe36e, 0xe36e, 0x203095, 0x420bb5, 0xc25668db40) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xca27c505c8, 0xc25b8d0000, 0xe36e, 0xe36e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc25668db40, 0xc25b8d0000, 0xe36e, 0xe36e, 0x2480, 0x5aed, 0xc228ecee90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc073b24958, 0x2b41380, 0xc25668db40, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc073b24700, 0x2b48520, 0xca27c505c8, 0x5, 0xca27c505c8, 0x2470) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc073b24700, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc073b24700, 0xc225e6bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc24422a900, 0xc225e6bab2, 0x254e, 0x254e, 0x2470, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc14a10bf80, 0xc225e6bab2, 0x254e, 0x254e, 0x2470, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc23c15d520, 0xc225e6bab2, 0x254e, 0x254e, 0x2470, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc006618ec0, 0xc225e6bab2, 0x254e, 0x254e, 0x2470, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc006618ec0, 0xc225e6bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc006618f40, 0xc225e6bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc006618f40, 0xc225e66000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0636dfb60, 0xc0636dfc20, 0xc0636dfc80, 0xc2206abc80, 0xc1e9d93260, 0xc12a8244d0, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4084803 [runnable]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc198ff8a80, 0xc22c7b0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc196b166c0, 0xc22c7b0000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc20f761ce0, 0xc22c7b0000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0fda1fa40, 0x7f59e0599060, 0xc20f761ce0, 0xc22c7b0000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0fda1fa40, 0x7f59e0599060, 0xc20f761ce0, 0xc22c7b0000, 0x8000, 0x8000, 0x2b6fc80, 0xc20f761ce0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1e07de8c0, 0x2b44ea0, 0xc0fda1fa40, 0xc22c7b0000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x56594bfb07bb7601, 0x17e3075b349f05d6, 0xc115bbe880, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x56594bfb07bb7601, 0x17e3075b349f05d6, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x56594bfb07bb7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc004abf800, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4522980 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc4f1083800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4084489 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1b69ccc00, 0xc21a218000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc1eddcf800, 0xc21a218000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc20deb1a70, 0xc21a218000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc102819310, 0x7f59e0599060, 0xc20deb1a70, 0xc21a218000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc102819310, 0x7f59e0599060, 0xc20deb1a70, 0xc21a218000, 0x8000, 0x8000, 0x2b6fc80, 0xc20deb1a70, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc006e3ab40, 0x2b44ea0, 0xc102819310, 0xc21a218000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xdd2142fb2cbd7601, 0x6f99466627d38173, 0xc1bdd68ec0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xdd2142fb2cbd7601, 0x6f99466627d38173, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xdd2142fb2cbd7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0054ca000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4084488 [runnable]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1aa0cfb00, 0xc224632000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e49ae40, 0xc224632000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc1a1b7dcb0, 0xc224632000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc112ec6b40, 0x7f59e0599060, 0xc1a1b7dcb0, 0xc224632000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc112ec6b40, 0x7f59e0599060, 0xc1a1b7dcb0, 0xc224632000, 0x8000, 0x8000, 0x2b6fc80, 0xc1a1b7dcb0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1e07de680, 0x2b44ea0, 0xc112ec6b40, 0xc224632000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x29e8bbfb721c7701, 0x15a357751cb03b9f, 0xc113afc380, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x29e8bbfb721c7701, 0x15a357751cb03b9f, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x29e8bbfb721c7701, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc004358900, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4582302 [select]: | |
| net/http.(*persistConn).readLoop(0xc0fa6fe480) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4582838 [select]: | |
| net/http.(*persistConn).readLoop(0xc0f7ded7a0) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4566155 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0ab132a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4515974 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc027f04cd0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d9dfc4e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0de578000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4561274 [select]: | |
| net/http.(*persistConn).writeLoop(0xc27abf85a0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4570183 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00d68dc04, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00d68db80, 0xc0bad590b4, 0xb, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294 | |
| github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc23f3997d0, 0xc0e77e0ab0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1693 +0x6f0 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc003b64ec0, 0xc0546f37c0, 0x3, 0x3, 0xc0b597b320, 0xc0b597b380, 0xc0b597b3e0, 0xc1379a6060, 0xc0f061a8a0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7 | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0077aba70, 0xc003b64ec0, 0xc149288900, 0xc0546f37c0, 0x3, 0x3, 0xc065e39800, 0x2b44f80, 0xc0e77e0c00, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0e77e0bd0, 0xc000233260) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4511299 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc167bf0900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582711 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f41e47e0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4582834 [select]: | |
| net/http.(*persistConn).readLoop(0xc10297e000) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4581516 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc1863eb080, 0xa, 0x2b99c80, 0xc004d641c0, 0xc1926117a0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 4582216 [select]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1da35ade0, 0xc1da35ae40, 0xc1da35aea0, 0xc009b1d580, 0xc1a1f8def0, 0xc0df870840, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4582301 [select]: | |
| net/http.(*persistConn).writeLoop(0xc9aa80f680) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4515567 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1437f97c0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1dd46eba0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc19caf6f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4577211 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc15680f900, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc246348cc0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc52a6fd200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4514790 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b8e60, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0c0910480, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc52a933e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4515930 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59c9f5ba40, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc166879098, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc166879080, 0xc2473da000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc166879080, 0xc2473da000, 0x8000, 0x8000, 0x400000801, 0x1, 0x8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1505a40e8, 0xc2473da000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc19149ea20, 0xc1df964498, 0x9, 0x9, 0xc2473da034, 0x4, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc19149ea20, 0xc1df964498, 0x9, 0x9, 0x9, 0xc000892dc8, 0x40cfa8, 0x20) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc1df964498, 0x9, 0x9, 0x2b41140, 0xc19149ea20, 0x0, 0xc200000000, 0x2124f40, 0xc123f98470) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc1df964460, 0xc123f98460, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00bc80d80, 0xc23ec67680, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc00bc80d80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc00bc80d80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4491184 [semacquire]: | |
| sync.runtime_Semacquire(0xc245857870) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc245857868) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc245857860, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc23e33dbf0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc147c57c20, 0x255e040, 0x18, 0xc2460b93f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc182805b00, 0x2bbccc0, 0xc18585a000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc182805b00, 0x7f63bf19f960, 0xc14837f080, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc182805b00, 0x2bbbd60, 0xc14837f080, 0xc000863100, 0xc239f45688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc185855f60, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc185855f40, 0xc185855de0, 0x26b3908, 0xc185855f40, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc185855f40, 0x2b99d40, 0xc147c57ad0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc185855f20, 0xc185855de0, 0xc185855e00, 0x7f63e7430a30, 0xc0231bed80) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc185855f20, 0xc185855de0, 0xc185855e00, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc185855f20, 0x2b99d40, 0xc147c57aa0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0xc185855de0, 0xc185855e20, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0x1, 0xc185855da0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0xc185855de0, 0xc185855e40, 0x19049ca, 0xb59d44573248) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0x203061, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0xc185855de0, 0xc185855e60, 0xc185855de0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0xc0c1dc4c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0xc185855de0, 0x26b3908, 0x2b99d40, 0xc147c57a40) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876200, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876200, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0182a84c0, 0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4084484 [syscall]: | |
| syscall.Syscall(0x4a, 0x249, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/syscall/asm_linux_amd64.s:18 +0x5 | |
| syscall.Fsync(0x249, 0x1, 0x14) | |
| /usr/local/go/src/syscall/zsyscall_linux_amd64.go:413 +0x40 | |
| internal/poll.(*FD).Fsync(0xc13fee3620, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_fsync_posix.go:17 +0x92 | |
| os.(*File).Sync(0xc151b91520, 0x0, 0x0) | |
| /usr/local/go/src/os/file_posix.go:113 +0x4c | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xc0ef16aae2b87601, 0x2b82198c9bb43095, 0xc1338effc0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:116 +0x421 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xc0ef16aae2b87601, 0x2b82198c9bb43095, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xc0ef16aae2b87601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0036a2b00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4474340 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0d10efa90, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d6757320, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1918b0600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4084490 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1e93b1e00, 0xc2110f8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc147ddb920, 0xc2110f8000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc2079cb5f0, 0xc2110f8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0b3b094a0, 0x7f59e0599060, 0xc2079cb5f0, 0xc2110f8000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0b3b094a0, 0x7f59e0599060, 0xc2079cb5f0, 0xc2110f8000, 0x8000, 0x8000, 0x2b6fc80, 0xc2079cb5f0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc004d14600, 0x2b44ea0, 0xc0b3b094a0, 0xc2110f8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xaca6514cb73a7601, 0x1d35228a8b6b854, 0xc113353580, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xaca6514cb73a7601, 0x1d35228a8b6b854, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xaca6514cb73a7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0010c8b00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4578952 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc079945680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4519239 [semacquire]: | |
| sync.runtime_Semacquire(0xc287ebdd50) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc287ebdd48) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc287ebdd40, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc28ab641b0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc131f367e0, 0x255e040, 0x18, 0xc2401353f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1d70ea660, 0x2bbccc0, 0xc0e07f5240, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1d70ea660, 0x7f63bf19f960, 0xc24318a320, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1d70ea660, 0x2bbbd60, 0xc24318a320, 0xc000863100, 0xc240135688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc0e07f51a0, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0e07f5180, 0xc0e07f5020, 0x26b3908, 0xc0e07f5180, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0e07f5180, 0x2b99d40, 0xc131f36690) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0e07f5160, 0xc0e07f5020, 0xc0e07f5040, 0x7f63e74235d8, 0xc0bc7c8300) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0e07f5160, 0xc0e07f5020, 0xc0e07f5040, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0e07f5160, 0x2b99d40, 0xc131f36660) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0xc0e07f5020, 0xc0e07f5060, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0x1, 0xc0e07f4fe0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0xc0e07f5020, 0xc0e07f5080, 0x25b4f230, 0xb5b5daf8b4c0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0x203038, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0xc0e07f5020, 0xc0e07f50a0, 0xc0e07f5020, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0xc0fb153c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0xc0e07f5020, 0x26b3908, 0x2b99d40, 0xc131f36600) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc106a9d600, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc106a9d600, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc126847560, 0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc106a9d600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4474342 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5a90be5a48, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc239f57b18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc239f57b00, 0xc1a0272000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc239f57b00, 0xc1a0272000, 0x8000, 0x8000, 0x4647d0, 0xc166433c80, 0x4) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc151594118, 0xc1a0272000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1d6757260, 0xc1febc10d8, 0x9, 0x9, 0xc0d3534e98, 0x461d40, 0xc0d3534e40) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1d6757260, 0xc1febc10d8, 0x9, 0x9, 0x9, 0xc0f4bc9d01, 0x10000000040cfa8, 0xc0d10efaa0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc1febc10d8, 0x9, 0x9, 0x2b41140, 0xc1d6757260, 0x0, 0x0, 0x0, 0xc0d10efa90) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc1febc10a0, 0xc25c2e6310, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1918b0600, 0xc1d7dc8390, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1918b0600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1918b0600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4581666 [running]: | |
| goroutine running on other thread; stack unavailable | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4513289 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5886ec7298, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1074ca118, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1074ca100, 0xc0e956c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1074ca100, 0xc0e956c000, 0x8000, 0x8000, 0x20309d, 0x420bb5, 0xc2747746e0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc000d54030, 0xc0e956c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc19679f200, 0xc0f0814ab8, 0x9, 0x9, 0xc005ac0000, 0x1007f63e74357c0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc19679f200, 0xc0f0814ab8, 0x9, 0x9, 0x9, 0x601742dd, 0xc14a0bedc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0f0814ab8, 0x9, 0x9, 0x2b41140, 0xc19679f200, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f0814a80, 0xc076bf4760, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc02f0d4600, 0xc00c2cf0e0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc02f0d4600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc02f0d4600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4515944 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0d62e1590, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d8e1e2a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc028eec180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4388231 [semacquire, 4 minutes]: | |
| sync.runtime_Semacquire(0xc0f5e21e70) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0f5e21e68) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0f5e21e60, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc0ff4119b0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc0f5e21da0, 0x255e040, 0x18, 0xc0f1f2f3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc0f5e42780, 0x2bbccc0, 0xc068e64a60, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc0f5e42780, 0x7f63bf19f960, 0xc0f5e6c200, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc0f5e42780, 0x2bbbd60, 0xc0f5e6c200, 0xc000863100, 0xc0d956b688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc068e649c0, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc068e649a0, 0xc068e64840, 0x26b3908, 0xc068e649a0, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc068e649a0, 0x2b99d40, 0xc0f5e21c50) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc068e64980, 0xc068e64840, 0xc068e64860, 0x7f63e7422f10, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc068e64980, 0xc068e64840, 0xc068e64860, 0x2b99d01, 0xc068e64980) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc068e64980, 0x2b99d40, 0xc0f5e21c20) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0xc068e64840, 0xc068e64880, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0x1, 0xc068e64800) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0xc068e64840, 0xc068e648a0, 0x1d76e66a, 0xb579c0bb14e5) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0x20301a, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x40c700, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0xc068e64840, 0xc068e648c0, 0xc068e64840, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0xc12b027c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0xc068e64840, 0x26b3908, 0x2b99d40, 0xc0f5e21bc0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0158e2480, 0xc0f5e0bc00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0158e2480, 0xc0f5e0bc00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0b0cfd630, 0xc0002c3380, 0x2bc6d40, 0xc0158e2480, 0xc0f5e0bc00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4582840 [select]: | |
| net/http.(*persistConn).readLoop(0xc24422a7e0) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4516668 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f63bf05ed88, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0d4a11b18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0d4a11b00, 0xc249970000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0d4a11b00, 0xc249970000, 0x8000, 0x8000, 0x20301d, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0b7aa0640, 0xc249970000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc09f936960, 0xc12d667ed8, 0x9, 0x9, 0xc009779880, 0x1007f63e743bca0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc09f936960, 0xc12d667ed8, 0x9, 0x9, 0x9, 0x601742e3, 0xc079ecbdc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc12d667ed8, 0x9, 0x9, 0x2b41140, 0xc09f936960, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc12d667ea0, 0xc0db948080, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc2fd38d200, 0xc24994faa0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc2fd38d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc2fd38d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4566156 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f588a55e480, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0b4977a18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0b4977a00, 0xc2462c0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0b4977a00, 0xc2462c0000, 0x8000, 0x8000, 0x400000801, 0xc000000001, 0x8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc00df12120, 0xc2462c0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1acbcfce0, 0xc11853d8b8, 0x9, 0x9, 0xc1ec071740, 0xc0a9813df7, 0xc0a9813d80) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1acbcfce0, 0xc11853d8b8, 0x9, 0x9, 0x9, 0x406594, 0xc083742480, 0xc0a9813df7) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc11853d8b8, 0x9, 0x9, 0x2b41140, 0xc1acbcfce0, 0x0, 0xc000000000, 0x2124f40, 0xc0f44c47b8) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc11853d880, 0xc0f44c4800, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0ab132a80, 0xc1db6c9ad0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0ab132a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0ab132a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4389190 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c51d1e0, 0xc075ccc5f0, 0x8, 0xc081dd5ad0, 0x28, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc197c68440, 0xc09d4d34a0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc197c68440, 0xc4168e2048, 0x1, 0x1, 0xc0b58b21e0, 0xd, 0x1, 0xc009779880, 0x7f63e743b5d8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc197c68440, 0xc100490720, 0xc4168e2048, 0x1, 0x1, 0xc1f17c4000, 0x2b44f80, 0xc1f17b26c0, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc0284e8da0, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1f17b2690, 0xc0005abab0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4514798 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5881ab49f0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc238281c18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc238281c00, 0xc234290000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc238281c00, 0xc234290000, 0x8000, 0x8000, 0x4647d0, 0xc00c4a7380, 0x4) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0060fa460, 0xc234290000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc0c0910a80, 0xc0460c3378, 0x9, 0x9, 0xc22f1f54f8, 0x461d40, 0xc22f1f54a0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc0c0910a80, 0xc0460c3378, 0x9, 0x9, 0x9, 0xc0594e7d01, 0x10000000040cfa8, 0xc02d3b92d0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0460c3378, 0x9, 0x9, 0x2b41140, 0xc0c0910a80, 0x0, 0x0, 0x0, 0xc02d3b92c0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c3340, 0xc114c7b480, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1a3430780, 0xc22f1dba40, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1a3430780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1a3430780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4508535 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f63c032f5c0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc23a318098, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc23a318080, 0xc2210c9000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc23a318080, 0xc2210c9000, 0x1000, 0x1000, 0x4378cc, 0xc0fe5b6b58, 0x4647d0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc9f9b34098, 0xc2210c9000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| net/http.(*persistConn).Read(0xc0deef0a20, 0xc2210c9000, 0x1000, 0x1000, 0xc093906780, 0xc0fe5b6c58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc1da7932c0) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1da7932c0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc19ef33b60) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0deef0a20) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4582525 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc0556e3b60, 0xa, 0x2b99c80, 0xc00702a880, 0xc2079cb920) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 4565088 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f588d6e5898, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00d100a18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00d100a00, 0xc2910f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc00d100a00, 0xc2910f4000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000071) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc9f9b34388, 0xc2910f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc11a5767e0, 0xc072428818, 0x9, 0x9, 0xc154428d40, 0x457243, 0xb5b93bba3c7f) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc11a5767e0, 0xc072428818, 0x9, 0x9, 0x9, 0xbffdee99c860543a, 0x15b79543d4d, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc072428818, 0x9, 0x9, 0x2b41140, 0xc11a5767e0, 0x0, 0x0, 0xc1fd836e20, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0724287e0, 0xc1fd836e20, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1ecc2c480, 0xc2910d0540, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1ecc2c480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1ecc2c480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4511256 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1245db680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582644 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f5887ee0150, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1ea0b9498, 0x72, 0xe100, 0xe1c1, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1ea0b9480, 0xc25b83c000, 0xe1c1, 0xe1c1, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1ea0b9480, 0xc25b83c000, 0xe1c1, 0xe1c1, 0x203092, 0x420bb5, 0xc2495d8de0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc150ab4d00, 0xc25b83c000, 0xe1c1, 0xe1c1, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc2495d8de0, 0xc25b83c000, 0xe1c1, 0xe1c1, 0x3360, 0x5940, 0xc1273f6e90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc07f222258, 0x2b41380, 0xc2495d8de0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc07f222000, 0x2b48520, 0xc150ab4d00, 0x5, 0xc150ab4d00, 0x3350) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc07f222000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc07f222000, 0xc1a1b6b000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc9aa80f680, 0xc1a1b6b000, 0x1000, 0x1000, 0x3350, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc23e49a180, 0xc20e237fa2, 0x5e, 0x5e, 0x3350, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.(*LimitedReader).Read(0xc23b0f8240, 0xc20e237fa2, 0x5e, 0x5e, 0x3350, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc00130c780, 0xc20e237fa2, 0x5e, 0x5e, 0x3350, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc00130c780, 0xc20e237fa2, 0x5e, 0x5e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc00130c7c0, 0xc20e237fa2, 0x5e, 0x5e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc00130c7c0, 0xc20e230000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc2654fa6c0, 0xc2654fa720, 0xc2654fa780, 0xc1ca6c62c0, 0xc20db2a9f0, 0xc085ceb1e0, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4578951 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0fe80c000, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19958b0e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc079945680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4578949 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1e65f3800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4515568 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc19caf6f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4559555 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00d68dc04, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00d68db80, 0xc1f2d70c84, 0xb, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294 | |
| github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc1f01ed7d0, 0xc287ebdc20, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1693 +0x6f0 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc005a83900, 0xc28b153b00, 0x3, 0x3, 0xc1335a85f0, 0xd, 0x1, 0xc222d2dc00, 0x7f63e74390f8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7 | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0077aba70, 0xc005a83900, 0xc1358474a0, 0xc28b153b00, 0x3, 0x3, 0xc1d70ea660, 0x2b44f80, 0xc287ebdd70, 0x8c, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc2837a3880, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc287ebdd40, 0xc14ebd24d0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4084806 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc21558cc00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257ace030, 0x25, 0x19cf51f4, 0x1c5f43a, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257ace030, 0x25, 0x19cf51f4, 0x1c5f43a, 0xc1e8a935c0, 0x40cfa8, 0x60, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257ace030, 0x25, 0x19cf51f4, 0x1c5f43a, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257ace030, 0x25, 0x19cf51f4, 0x1c5f43a, 0x0, 0x4000105, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc000c90ae0, 0x2b99c80, 0xc000e9e080, 0xc257ace030, 0x25, 0x19cf51f4, 0x1c5f43a, 0x0, 0x7f59e0599060, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000270c00, 0x2b99c80, 0xc000e9e080, 0xc254da6d60, 0x20, 0x19cf51f4, 0x1c5f43a, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc00266bf40, 0x2b44ea0, 0xc114f2ae60, 0xc210bdc000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x425ec0b5923b7601, 0x158267717d66e356, 0xc12deb8f00, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x425ec0b5923b7601, 0x158267717d66e356, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x425ec0b5923b7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc002343700, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4561862 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59a29f3388, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc28cf02e98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc28cf02e80, 0xc13f998000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc28cf02e80, 0xc13f998000, 0x8000, 0x8000, 0x20307a, 0x420bb5, 0xc1ea4b6a00) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc46e0a2028, 0xc13f998000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1908b5aa0, 0xc112bf4038, 0x9, 0x9, 0xc001e05180, 0x1007f63e74310f8, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1908b5aa0, 0xc112bf4038, 0x9, 0x9, 0x9, 0x601742ea, 0xc160c70dc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc112bf4038, 0x9, 0x9, 0x2b41140, 0xc1908b5aa0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc112bf4000, 0xc2561c5800, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0ab2bbe00, 0xc28f847470, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0ab2bbe00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0ab2bbe00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4510186 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc19966bb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582808 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc092731ec0, 0xa, 0x2b99c80, 0xc004f95480, 0xc23b892f00) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 4514789 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59d957e260, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1c30b4298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1c30b4280, 0xc233548000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1c30b4280, 0xc233548000, 0x8000, 0x8000, 0x203048, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1f017c018, 0xc233548000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc2442d73e0, 0xc0460c30d8, 0x9, 0x9, 0xc1d4e89c00, 0x1007f63e741a848, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc2442d73e0, 0xc0460c30d8, 0x9, 0x9, 0x9, 0x601742e3, 0xc07159bdc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0460c30d8, 0x9, 0x9, 0x2b41140, 0xc2442d73e0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c30a0, 0xc260cdd5c0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc52a933c80, 0xc22f1db410, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc52a933c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc52a933c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4581580 [running]: | |
| goroutine running on other thread; stack unavailable | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4517322 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59c9c987d8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1c30b4418, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1c30b4400, 0xc202fbe000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1c30b4400, 0xc202fbe000, 0x8000, 0x8000, 0x800010601, 0x0, 0x10601000000f8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1f017c030, 0xc202fbe000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1df74d320, 0xc0ce7d9618, 0x9, 0x9, 0xc038e1dd40, 0x457243, 0xb5b93caf9027) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1df74d320, 0xc0ce7d9618, 0x9, 0x9, 0x9, 0xbffdee99c94efb94, 0x15b7a42e4dc, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0ce7d9618, 0x9, 0x9, 0x2b41140, 0xc1df74d320, 0x0, 0x0, 0xc0a0e95f00, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0ce7d95e0, 0xc0a0e95f00, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1a2f8e780, 0xc23f952f90, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1a2f8e780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1a2f8e780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4084486 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc19966a300, 0xc225e66000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e84e5a0, 0xc225e66000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc1e8b47ad0, 0xc225e66000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0b8635220, 0x7f59e0599060, 0xc1e8b47ad0, 0xc225e66000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0b8635220, 0x7f59e0599060, 0xc1e8b47ad0, 0xc225e66000, 0x8000, 0x8000, 0x2b6fc80, 0xc1e8b47ad0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1e07de4c0, 0x2b44ea0, 0xc0b8635220, 0xc225e66000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x2e1ca4e423a27601, 0x194080b5c8660802, 0xc11632e400, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x2e1ca4e423a27601, 0x194080b5c8660802, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x2e1ca4e423a27601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc002413f00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4582841 [select]: | |
| net/http.(*persistConn).writeLoop(0xc24422a7e0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4513605 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f58877a53c8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc06ce7e518, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc06ce7e500, 0xc103990000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc06ce7e500, 0xc103990000, 0x8000, 0x8000, 0x4647d0, 0xc00109f080, 0x4) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc143d460c0, 0xc103990000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc10cfe4f00, 0xc1b5e6c578, 0x9, 0x9, 0xc2469ad978, 0x461d40, 0xc2469ad920) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc10cfe4f00, 0xc1b5e6c578, 0x9, 0x9, 0x9, 0xc079eccd01, 0x10000000040cfa8, 0xc175158d80) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc1b5e6c578, 0x9, 0x9, 0x2b41140, 0xc10cfe4f00, 0x0, 0x0, 0x0, 0xc175158d70) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc1b5e6c540, 0xc201589ff0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc10a44af00, 0xc24500a5d0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc10a44af00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc10a44af00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4582745 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f84707e0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4517320 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0df651f90, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1cb011f80, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1a2f8e780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4514915 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1503a0500, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc198be6c60, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1a3430f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4513288 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc02f0d4600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4514701 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b8870, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc2442d6960, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc21d477c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4513287 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc157075040, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19679f2c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc02f0d4600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4582635 [select]: | |
| net/http.(*persistConn).readLoop(0xc2386e6a20) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4517361 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f58927f8ee0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc06ce7e498, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc06ce7e480, 0xc147c9a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc06ce7e480, 0xc147c9a000, 0x8000, 0x8000, 0x203079, 0x420bb5, 0xc1e7bacee0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc143d460b8, 0xc147c9a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc100420060, 0xc0c822c658, 0x9, 0x9, 0xc009778a80, 0x1007f63e7434368, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc100420060, 0xc0c822c658, 0x9, 0x9, 0x9, 0x601742ea, 0xc038e20dc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0c822c658, 0x9, 0x9, 0x2b41140, 0xc100420060, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0c822c620, 0xc1f810baa0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc19caf6f00, 0xc24726b0b0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc19caf6f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc19caf6f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4556670 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc28317d9f0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc191227380, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc183057e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4582712 [runnable]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc06ed7ab40, 0xc06ed7ade0, 0xc06ed7ae40, 0xc00d5e55c0, 0xc23ad14450, 0xc0099ab4a0, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4514690 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0bdda9c70, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc2442d6060, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc21d477500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4084496 [syscall]: | |
| syscall.Syscall(0x4a, 0x1d8, 0x0, 0x0, 0x4edbd3, 0xc1d72be380, 0xc249195778) | |
| /usr/local/go/src/syscall/asm_linux_amd64.s:18 +0x5 | |
| syscall.Fsync(0x1d8, 0x2ade801, 0x0) | |
| /usr/local/go/src/syscall/zsyscall_linux_amd64.go:413 +0x40 | |
| internal/poll.(*FD).Fsync(0xc193e528a0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_fsync_posix.go:17 +0x92 | |
| os.(*File).Sync(0xc029070e30, 0x3b, 0x8000) | |
| /usr/local/go/src/os/file_posix.go:113 +0x4c | |
| github.com/thanos-io/thanos/pkg/block/indexheader.newBinaryWriter(0xc1d72be380, 0x3b, 0xc2557ba000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:269 +0x22b | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xb15fcd643c807601, 0x4bc47f928c0a90ae, 0xc175b71200, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:82 +0x18b | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xb15fcd643c807601, 0x4bc47f928c0a90ae, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xb15fcd643c807601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc003dbae00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4582515 [runnable]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0167b2e40, 0xc0167b2ea0, 0xc0167b2f00, 0xc0d318b390, 0xc2079ca990, 0xc133c34580, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4513532 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f63bf30f008, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc182359898, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc182359880, 0xc17e9c2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc182359880, 0xc17e9c2000, 0x8000, 0x8000, 0x800010601, 0x7f5900000000, 0x106010000006e) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0054060b8, 0xc17e9c2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc18aab5740, 0xc0dceeddf8, 0x9, 0x9, 0xc13e423d40, 0x457243, 0xb5b9026248b4) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc18aab5740, 0xc0dceeddf8, 0x9, 0x9, 0x9, 0xbffdee9985965cd7, 0x15b3aef7be7, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0dceeddf8, 0x9, 0x9, 0x2b41140, 0xc18aab5740, 0x0, 0x0, 0xc2594b4260, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0dceeddc0, 0xc2594b4260, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc078bd3200, 0xc11b1a3740, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc078bd3200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc078bd3200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4521104 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc12e0d6300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4578957 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59d9b66378, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc167221a98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc167221a80, 0xc1ad3da000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc167221a80, 0xc1ad3da000, 0x8000, 0x8000, 0x10401, 0x0, 0x1040102ade864) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc142924920, 0xc1ad3da000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1ea5010e0, 0xc06785a578, 0x9, 0x9, 0xc00596d180, 0x7f63e7434a30, 0xc04a5d1380) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1ea5010e0, 0xc06785a578, 0x9, 0x9, 0x9, 0xa6fc25, 0xc1136af5f0, 0xc1136a0004) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc06785a578, 0x9, 0x9, 0x2b41140, 0xc1ea5010e0, 0x0, 0x0, 0xc1136af5f0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc06785a540, 0xc1136af5f0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc079945980, 0xc19a278c30, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc079945980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc079945980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4516940 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc20dacfae0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1fd070060, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc003a9e900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4462380 [semacquire]: | |
| sync.runtime_Semacquire(0xc1222108b0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1222108a8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1222108a0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1dc0d7b60) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc14bcc4870, 0x255e040, 0x18, 0xc2461153f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1549fe360, 0x2bbccc0, 0xc18983e260, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1549fe360, 0x7f63bf19f960, 0xc16eeb35d0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1549fe360, 0x2bbbd60, 0xc16eeb35d0, 0xc000863100, 0xc238491688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc18b801fc0, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc18b801fa0, 0xc18b801e40, 0x26b3908, 0xc18b801fa0, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc18b801fa0, 0x2b99d40, 0xc13ffe2570) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18b801f80, 0xc18b801e40, 0xc18b801e60, 0x7f63e74250f8, 0xc054bff680) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18b801f80, 0xc18b801e40, 0xc18b801e60, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18b801f80, 0x2b99d40, 0xc13ffe2540) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0xc18b801e40, 0xc18b801e80, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0x1, 0xc18b801e00) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0xc18b801e40, 0xc18b801ea0, 0x1bd9e773, 0xb59cabd071f1) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0x203062, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0xc18b801e00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0xc18b801e40, 0xc18b801ec0, 0x203062, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0xc027804c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0xc18b801e40, 0x26b3908, 0x2b99d40, 0xc13ffe24e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1defd8100, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1defd8100, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0b41fb330, 0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1defd8100) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4562285 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc007cd0344, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc007cd02c0, 0xc1580704d0, 0x8, 0xc25a00bc50, 0x21, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc197c692c0, 0xc2003ed1e0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc197c692c0, 0xc00357c4c8, 0x1, 0x1, 0x2d850e0a0cd5be5, 0x6b4423492606b1b3, 0xeb462cad18d3a9f, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0040c6990, 0xc197c692c0, 0xc13edbc3c0, 0xc00357c4c8, 0x1, 0x1, 0xc13edbc1e0, 0x2b44f80, 0xc1ea93a480, 0x60, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1ea93a450, 0xc1e8aaf2d0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4582881 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc07f961ce0, 0xa, 0x2b99c80, 0xc1e07def00, 0xc225a73d10) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 4522979 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc039d83ef0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1f9321380, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc4f1083800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4582865 [select]: | |
| net/http.(*persistConn).writeLoop(0xc22ca94a20) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4486231 [select, 3 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1c1c82180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582720 [select]: | |
| net/http.(*persistConn).readLoop(0xc22ca94a20) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4582300 [select]: | |
| net/http.(*persistConn).readLoop(0xc9aa80f680) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4582521 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0fd6c8000) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4514914 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f63bf05e5a8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc238281c98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc238281c80, 0xc236b32000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc238281c80, 0xc236b32000, 0x8000, 0x8000, 0x4647d0, 0xc0b9286900, 0x4) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0060fa468, 0xc236b32000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc0c09119e0, 0xc0460c3458, 0x9, 0x9, 0xc22f1f5738, 0x461d40, 0xc22f1f56e0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc0c09119e0, 0xc0460c3458, 0x9, 0x9, 0x9, 0xc0594e8d01, 0x10000000040cfa8, 0xc1503a02e0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0460c3458, 0x9, 0x9, 0x2b41140, 0xc0c09119e0, 0x0, 0x0, 0x0, 0xc1503a02d0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c3420, 0xc0f0f4bb70, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1a3430c00, 0xc236b302a0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1a3430c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1a3430c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4514646 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc21d477200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4084493 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1c1797b00, 0xc224286000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc147ddb8c0, 0xc224286000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc2079cb4d0, 0xc224286000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0bf1485a0, 0x7f59e0599060, 0xc2079cb4d0, 0xc224286000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0bf1485a0, 0x7f59e0599060, 0xc2079cb4d0, 0xc224286000, 0x8000, 0x8000, 0x2b6fc80, 0xc2079cb4d0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc9f4139280, 0x2b44ea0, 0xc0bf1485a0, 0xc224286000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xd5aa35a62ec7601, 0x36cc35e8daaa0f55, 0xc115bc4240, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xd5aa35a62ec7601, 0x36cc35e8daaa0f55, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xd5aa35a62ec7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0073c2200, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4582668 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f59df27c0e8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1e9de4718, 0x72, 0xe000, 0xe0a5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1e9de4700, 0xc25bbde000, 0xe0a5, 0xe0a5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1e9de4700, 0xc25bbde000, 0xe0a5, 0xe0a5, 0x203094, 0x420bb5, 0xc25273e980) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc142924a10, 0xc25bbde000, 0xe0a5, 0xe0a5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc25273e980, 0xc25bbde000, 0xe0a5, 0xe0a5, 0x0, 0x41310b, 0xc145a15900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc066d7a258, 0x2b41380, 0xc25273e980, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc066d7a000, 0x2b48520, 0xc142924a10, 0x5, 0xc142924a10, 0x40c924) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc066d7a000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc066d7a000, 0xc23df34000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc163269d40, 0xc23df34000, 0x1000, 0x1000, 0xc0cc95a780, 0xc145a15c58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc0bad13e60) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc0bad13e60, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc029bd1a40) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc163269d40) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4557186 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc11159f080) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4517270 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc20dacfbd0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1f71bac00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc003a9ea80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4517321 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1a2f8e780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4084485 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc01c240600, 0xc210d60000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e49b1a0, 0xc210d60000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc1a1b7def0, 0xc210d60000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc11357b130, 0x7f59e0599060, 0xc1a1b7def0, 0xc210d60000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc11357b130, 0x7f59e0599060, 0xc1a1b7def0, 0xc210d60000, 0x8000, 0x8000, 0x2b6fc80, 0xc1a1b7def0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1a4218c40, 0x2b44ea0, 0xc11357b130, 0xc210d60000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xb3deb7395f797601, 0x6912f1f70bc8663, 0xc113352840, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xb3deb7395f797601, 0x6912f1f70bc8663, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xb3deb7395f797601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0029ddd00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4514917 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59c9f7d630, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc238281d18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc238281d00, 0xc236b62000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc238281d00, 0xc236b62000, 0x8000, 0x8000, 0x800010601, 0x0, 0x106010000004c) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0060fa470, 0xc236b62000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc0c0911b00, 0xc0460c3618, 0x9, 0x9, 0xc079ecad40, 0x457243, 0xb5b93bf2a674) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc0c0911b00, 0xc0460c3618, 0x9, 0x9, 0x9, 0xbffdee99c899aeb9, 0x15b798d97bc, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0460c3618, 0x9, 0x9, 0x2b41140, 0xc0c0911b00, 0x0, 0x0, 0xc0e6406980, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c35e0, 0xc0e6406980, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1a3430f00, 0xc236b30570, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1a3430f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1a3430f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4508843 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5889dba5b8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc06ce7e218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc06ce7e200, 0xc203e18000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc06ce7e200, 0xc203e18000, 0x8000, 0x8000, 0x203034, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc143d46070, 0xc203e18000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1d98a5c80, 0xc0f7ba02d8, 0x9, 0x9, 0xc22c4eca80, 0x1007f63e743f5d8, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1d98a5c80, 0xc0f7ba02d8, 0x9, 0x9, 0x9, 0x601742e2, 0xc0594e4dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0f7ba02d8, 0x9, 0x9, 0x2b41140, 0xc1d98a5c80, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f7ba02a0, 0xc0caf8a8e0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc4b81e0600, 0xc14938b920, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc4b81e0600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc4b81e0600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4084807 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1f0dbbe00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257ab8750, 0x25, 0x15803a8f, 0x1d2ece6, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257ab8750, 0x25, 0x15803a8f, 0x1d2ece6, 0xc23cc6f5c0, 0x40cfa8, 0x60, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257ab8750, 0x25, 0x15803a8f, 0x1d2ece6, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257ab8750, 0x25, 0x15803a8f, 0x1d2ece6, 0x0, 0x1000100, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc000c90ae0, 0x2b99c80, 0xc000e9e080, 0xc257ab8750, 0x25, 0x15803a8f, 0x1d2ece6, 0x0, 0x7f59e0599060, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000270c00, 0x2b99c80, 0xc000e9e080, 0xc253dc2f40, 0x20, 0x15803a8f, 0x1d2ece6, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc43967ff80, 0x2b44ea0, 0xc1271b6640, 0xc21b34c000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xdae8b344428f7601, 0x591094f781cff076, 0xc114726e40, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xdae8b344428f7601, 0x591094f781cff076, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xdae8b344428f7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0036a8300, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4467634 [semacquire]: | |
| sync.runtime_Semacquire(0xc1e00b9a20) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1e00b9a18) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1e00b9a10, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1edc66030) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc232a72f90, 0x255e040, 0x18, 0xc1e8c9b3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc2016bb560, 0x2bbccc0, 0xc1e66760e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc2016bb560, 0x7f63bf19f960, 0xc2016b9500, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc2016bb560, 0x2bbbd60, 0xc2016b9500, 0xc000863100, 0xc169933688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1e6676040, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e6676000, 0xc1b5375ea0, 0x26b3908, 0xc1e6676000, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e6676000, 0x2b99d40, 0xc232a72e40) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375fe0, 0xc1b5375ea0, 0xc1b5375ec0, 0x7f63e741f5d8, 0xc04a91cc00) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375fe0, 0xc1b5375ea0, 0xc1b5375ec0, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375fe0, 0x2b99d40, 0xc232a72e10) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0xc1b5375ea0, 0xc1b5375ee0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0x1, 0xc1b5375e60) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0xc1b5375ea0, 0xc1b5375f00, 0x2a336da9, 0xb57d4b897222) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0x20306d, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0xc1b5375ea0, 0xc1b5375f20, 0xc1b5375ea0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0xc2271e5c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0xc1b5375ea0, 0x26b3908, 0x2b99d40, 0xc232a72db0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469d00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469d00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0051527a0, 0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469d00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4514797 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1a3430780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4388342 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00410104c, 0x3f75800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc004101048) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc004101048) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc004100fd0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).IndexVersion(0xc004100fd0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:144 +0xa0 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1f7072f00, 0xc9ea0960d8, 0x1, 0x1, 0xc026b230c0, 0xd, 0x1, 0xc000e7c700, 0x7f63e741c368) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1639 +0xfbf | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc002a7a1b0, 0xc1f7072f00, 0xc0ff2141e0, 0xc9ea0960d8, 0x1, 0x1, 0xc1c9270d80, 0x2b44f80, 0xc0ff208000, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0f4f5bfb0, 0xc0095a88c0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4514913 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1a3430c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4511298 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc129f259a0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d8e720c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc167bf0900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4496797 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00c51d264, 0xc23886c200, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00c51d1e0, 0xc18e916c60, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294 | |
| github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc0070d77d0, 0xc1e00b98f0, 0x66, 0x3f96db8, 0x203011) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1713 +0x403 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc20a83d000, 0xc1ffe7c800, 0x2, 0x2, 0x203079, 0x417a5e, 0x7f59d8e90040, 0x203079, 0xbffdee95db925928) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7 | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc20a83d000, 0xc22ff38cc0, 0xc1ffe7c800, 0x2, 0x2, 0xc2016bb560, 0x2b44f80, 0xc1e00b9a40, 0xc00017c0b8, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1e00b9a10, 0xc1307d1f80) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4514785 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc21d477e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4555622 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59c9d0ce48, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc125066318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc125066300, 0xc1deb1e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc125066300, 0xc1deb1e000, 0x8000, 0x8000, 0x203079, 0x420bb5, 0xc1e7b8be80) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc9f024c3f8, 0xc1deb1e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc0056b9200, 0xc0f09056f8, 0x9, 0x9, 0xc1ed8d7880, 0x1007f63e7434368, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc0056b9200, 0xc0f09056f8, 0x9, 0x9, 0x9, 0x601742e8, 0xc227d33dc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0f09056f8, 0x9, 0x9, 0x2b41140, 0xc0056b9200, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f09056c0, 0xc1f810b740, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc081fa6900, 0xc23a5bb140, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc081fa6900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc081fa6900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4555620 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0d06739a0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc093978de0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc081fa6900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4582667 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0b6675440) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4521103 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0ea4f9f90, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1ecefa900, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc12e0d6300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4582524 [select]: | |
| net/http.(*persistConn).roundTrip(0xc163269d40, 0xc00702a9c0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc000ce5540, 0xc249c4e900, 0xc249f206c0, 0x40f967, 0xc000140a08) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc000ce5540, 0xc249c4e900, 0xc000ce5540, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc249c4e900, 0x2b485c0, 0xc000ce5540, 0x0, 0x0, 0x0, 0xc143d46958, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000bfaba0, 0xc249c4e900, 0x0, 0x0, 0x0, 0xc143d46958, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000bfaba0, 0xc249c4e900, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0556e3a40, 0xc0556e3aa0, 0xc0556e3b00, 0xc0ea676760, 0xc2079cb890, 0xc133c34c60, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4523809 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5992d88c90, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc21a409f18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc21a409f00, 0xc10ad9a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc21a409f00, 0xc10ad9a000, 0x8000, 0x8000, 0x203078, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1c67f4230, 0xc10ad9a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1d25b8de0, 0xc249c7e118, 0x9, 0x9, 0xc0013d0700, 0x1007f63e742c368, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1d25b8de0, 0xc249c7e118, 0x9, 0x9, 0x9, 0x601742e2, 0xc10d6e4dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc249c7e118, 0x9, 0x9, 0x2b41140, 0xc1d25b8de0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc249c7e0e0, 0xc21589e0c0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc12e0d6300, 0xc236f20c30, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc12e0d6300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc12e0d6300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4496830 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00c51d1e0, 0xc18e916c18, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294 | |
| github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc2414a97d0, 0xc243035380, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1713 +0x403 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1f8819380, 0xc214344370, 0x2, 0x2, 0x30, 0x0, 0xc003208000, 0xc0921ddae0, 0x30) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7 | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc1f8819380, 0xc23802c5a0, 0xc214344370, 0x2, 0x2, 0xc2016bb440, 0x2b44f80, 0xc2430354d0, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc2430354a0, 0xc00b2e0e70) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 3720229 [select]: | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0x1, 0x1050101) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:391 +0x2ee | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).SyncBlocks.func1(0x2b99c80, 0xc000e9e080, 0xc0001e6a20, 0x20, 0x225c080) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:139 +0x3f | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).syncUsersBlocks.func2(0xc1fc5dbfa0, 0xc008e14840, 0x26b38d8, 0x2b99c80, 0xc000e9e080, 0xc1fc5dbfb0, 0xc200f022e0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:186 +0x107 | |
| created by github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).syncUsersBlocks | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:182 +0x45d | |
| goroutine 4462480 [semacquire]: | |
| sync.runtime_Semacquire(0xc2430354b0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc2430354a8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc2430354a0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1e7c4fc50) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc232a72bd0, 0x255e040, 0x18, 0xc1ed8ff3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc2016bb440, 0x2bbccc0, 0xc1b5375c20, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc2016bb440, 0x7f63bf19f960, 0xc2016b9460, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc2016bb440, 0x2bbbd60, 0xc2016b9460, 0xc000863100, 0xc1ed187688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1b5375b80, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1b5375b60, 0xc1b53759e0, 0x26b3908, 0xc1b5375b60, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1b5375b60, 0x2b99d40, 0xc232a72a50) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375b40, 0xc1b53759e0, 0xc1b5375a00, 0x7f63e741f5d8, 0xc4eda5bc80) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375b40, 0xc1b53759e0, 0xc1b5375a00, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375b40, 0x2b99d40, 0xc232a72a20) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0xc1b53759e0, 0xc1b5375a20, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0x1, 0xc1b53759a0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0xc1b53759e0, 0xc1b5375a40, 0x2a324ece, 0xb57d4b885344) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0x20306d, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0xc1b53759e0, 0xc1b5375a60, 0xc1b53759e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0xc2271e5c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0xc1b53759e0, 0x26b3908, 0x2b99d40, 0xc232a729c0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469b00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469b00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0051527a0, 0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4376735 [semacquire, 4 minutes]: | |
| sync.runtime_Semacquire(0xc1f17b26a0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1f17b2698) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1f17b2690, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc100498420) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1f17b25d0, 0x255e040, 0x18, 0xc101df13f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1f17c4000, 0x2bbccc0, 0xc0eba36820, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1f17c4000, 0x7f63bf19f960, 0xc00db9a250, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1f17c4000, 0x2bbbd60, 0xc00db9a250, 0xc000863100, 0xc101b69688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc0eba36780, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0eba36760, 0xc0eba36600, 0x26b3908, 0xc0eba36760, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0eba36760, 0x2b99d40, 0xc1f17b2480) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0eba36740, 0xc0eba36600, 0xc0eba36620, 0x7f63e7428a30, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0eba36740, 0xc0eba36600, 0xc0eba36620, 0x2b99d01, 0xc0eba36740) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0eba36740, 0x2b99d40, 0xc1f17b2450) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0xc0eba36600, 0xc0eba36640, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0x7f599a973200, 0x20307c00000000) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0xc0eba36600, 0xc0eba36660, 0x1da2db7c, 0xb579c0e709fe) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0x20303a, 0xc0010000000000) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x7f59de527d00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0xc0eba36600, 0xc0eba36680, 0xc0eba36600, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0xc12884fc68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0xc0eba36600, 0x26b3908, 0x2b99d40, 0xc1f17b23f0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc09673ec00, 0xc1defd9700, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc09673ec00, 0xc1defd9700, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0eb34a3d0, 0xc0002c3380, 0x2bc6d40, 0xc09673ec00, 0xc1defd9700) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4514786 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5995457930, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc238281a98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc238281a80, 0xc233530000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc238281a80, 0xc233530000, 0x8000, 0x8000, 0x203062, 0x420bb5, 0xc188816820) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0060fa438, 0xc233530000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc2442d67e0, 0xc0460c2ff8, 0x9, 0x9, 0xc00c5740a0, 0xc06edf4d60, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc2442d67e0, 0xc0460c2ff8, 0x9, 0x9, 0x9, 0xc06edf4dc8, 0x40cfa8, 0xc02d3b8a60) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0460c2ff8, 0x9, 0x9, 0x2b41140, 0xc2442d67e0, 0x0, 0x0, 0x0, 0xc02d3b8a50) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c2fc0, 0xc25c2e6330, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc21d477e00, 0xc22f1db200, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc21d477e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc21d477e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4582836 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f59e088d530, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00cd7b718, 0x72, 0xdb00, 0xdb99, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00cd7b700, 0xc248fda000, 0xdb99, 0xdb99, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc00cd7b700, 0xc248fda000, 0xdb99, 0xdb99, 0x203094, 0x420bb5, 0xc25086b2e0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc150ab4cf0, 0xc248fda000, 0xdb99, 0xdb99, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc25086b2e0, 0xc248fda000, 0xdb99, 0xdb99, 0x0, 0x41310b, 0xc02ad7a900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc07a5073d8, 0x2b41380, 0xc25086b2e0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc07a507180, 0x2b48520, 0xc150ab4cf0, 0x5, 0xc150ab4cf0, 0x40c924) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc07a507180, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc07a507180, 0xc23e39a000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0fb8f86c0, 0xc23e39a000, 0x1000, 0x1000, 0xc0ca3477a0, 0xc02ad7ac58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc23e378780) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc23e378780, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc072dc6540) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0fb8f86c0) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4582669 [select]: | |
| net/http.(*persistConn).writeLoop(0xc163269d40) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4463557 [semacquire, 4 minutes]: | |
| sync.runtime_Semacquire(0xc237c704c0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc237c704b8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc237c704b0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc2389e6000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc237c703f0, 0x255e040, 0x18, 0xc1ef34b3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1ee23d4a0, 0x2bbccc0, 0xc0b013ea40, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1ee23d4a0, 0x7f63bf19f960, 0xc225d72e70, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1ee23d4a0, 0x2bbbd60, 0xc225d72e70, 0xc000863100, 0xc1ec633688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc0b013e940, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0b013e920, 0xc0b013e760, 0x26b3908, 0xc0b013e920, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0b013e920, 0x2b99d40, 0xc237c702a0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0b013e8e0, 0xc0b013e760, 0xc0b013e780, 0x7f63e743a180, 0xc4edda2180) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0b013e8e0, 0xc0b013e760, 0xc0b013e780, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0b013e8e0, 0x2b99d40, 0xc237c70270) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0xc0b013e760, 0xc0b013e7a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0x1, 0xc0b013e720) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0xc0b013e760, 0xc0b013e7c0, 0x39027743, 0xb57b064c97c1) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0x20302c, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0xc0b013e760, 0xc0b013e7e0, 0xc0b013e760, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0xc0f0ed7c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0xc0b013e760, 0x26b3908, 0x2b99d40, 0xc237c70210) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc00f65e300, 0xc1ee217400, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc00f65e300, 0xc1ee217400, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc019c06460, 0xc0002c3380, 0x2bc6d40, 0xc00f65e300, 0xc1ee217400) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4510187 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59943af680, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1ebaee018, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1ebaee000, 0xc0cebc8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1ebaee000, 0xc0cebc8000, 0x8000, 0x8000, 0x203063, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc004ffa020, 0xc0cebc8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc22a263d40, 0xc12d666498, 0x9, 0x9, 0xc0062d0300, 0x1007f63e743e848, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc22a263d40, 0xc12d666498, 0x9, 0x9, 0x9, 0x601742dc, 0xc22c8bddc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc12d666498, 0x9, 0x9, 0x2b41140, 0xc22a263d40, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc12d666460, 0xc08ed76900, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc19966bb00, 0xc23e2b1f80, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc19966bb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc19966bb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4582636 [select]: | |
| net/http.(*persistConn).writeLoop(0xc2386e6a20) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4084801 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1aa0a9500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257835260, 0x25, 0x22dcf307, 0x287a7c5, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257835260, 0x25, 0x22dcf307, 0x287a7c5, 0xc29037f5c0, 0x40cfa8, 0x60, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257835260, 0x25, 0x22dcf307, 0x287a7c5, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257835260, 0x25, 0x22dcf307, 0x287a7c5, 0x0, 0x2000107, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc000c90ae0, 0x2b99c80, 0xc000e9e080, 0xc257835260, 0x25, 0x22dcf307, 0x287a7c5, 0x0, 0x7f59e0599060, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000270c00, 0x2b99c80, 0xc000e9e080, 0xc0c94f36e0, 0x20, 0x22dcf307, 0x287a7c5, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc1a42187c0, 0x2b44ea0, 0xc18249caf0, 0xc20eba8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x8e6caa1f6f17601, 0x9d4aa4ded66b726f, 0xc143f837c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x8e6caa1f6f17601, 0x9d4aa4ded66b726f, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x8e6caa1f6f17601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc006837b00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4577213 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59c9f7d710, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc06bec8a98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc06bec8a80, 0xc1b2e50000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc06bec8a80, 0xc1b2e50000, 0x8000, 0x8000, 0x203079, 0x420bb5, 0xc1e7baca40) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc005f6b110, 0xc1b2e50000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc2608022a0, 0xc29067d0d8, 0x9, 0x9, 0xc001e05180, 0x1007f63e7434368, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc2608022a0, 0xc29067d0d8, 0x9, 0x9, 0x9, 0x601742ea, 0xc165f05dc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc29067d0d8, 0x9, 0x9, 0x2b41140, 0xc2608022a0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc29067d0a0, 0xc1f810ba20, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc52a6fd200, 0xc1117f5a40, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc52a6fd200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc52a6fd200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4465623 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c51d1e0, 0xc19785e2b8, 0x8, 0xc01a468960, 0x28, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc2228596c0, 0xc0b5eadfc0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc2228596c0, 0xc004ffa1c8, 0x1, 0x1, 0xaad796cb0b2e042, 0xf0a1fce18348224, 0x60aad9cb3a5ef05b, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc2228596c0, 0xc23b081a40, 0xc004ffa1c8, 0x1, 0x1, 0xc22276ec00, 0x2b44f80, 0xc216327fb0, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc02c2a0d60, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc216327f80, 0xc222865d50) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4582147 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc007cd0344, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).IndexVersion(0xc007cd02c0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:141 +0x1f7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0049f9000, 0xc084f17b90, 0x2, 0x2, 0x28bcc65066b49d01, 0x28784e46ada33c4d, 0x5a52b691f65db313, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1639 +0xfbf | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0040c6990, 0xc0049f9000, 0xc046eefd40, 0xc084f17b90, 0x2, 0x2, 0xc197b33380, 0x2b44f80, 0xc1e8866780, 0xc000cf0428, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1e8866750, 0xc1e8b40230) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4084491 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc134499b00, 0xc20e230000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc11c487320, 0xc20e230000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc173f54b70, 0xc20e230000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc128a48c30, 0x7f59e0599060, 0xc173f54b70, 0xc20e230000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc128a48c30, 0x7f59e0599060, 0xc173f54b70, 0xc20e230000, 0x8000, 0x8000, 0x2b6fc80, 0xc173f54b70, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0026ad240, 0x2b44ea0, 0xc128a48c30, 0xc20e230000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xf6f451aa76287701, 0x9dc833dc45eea49f, 0xc12deb9840, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xf6f451aa76287701, 0x9dc833dc45eea49f, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xf6f451aa76287701, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0088b8c00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4514793 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b9040, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0c0910c60, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1a3430180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4561860 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc28c43a870, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc079aead80, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0ab2bbe00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4582114 [select]: | |
| net/http.(*persistConn).writeLoop(0xc2433439e0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4508842 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc4b81e0600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4561339 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5884b38c40, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc23562ac98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc23562ac80, 0xc1debe4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc23562ac80, 0xc1debe4000, 0x8000, 0x8000, 0x400000801, 0xc100000001, 0x8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1505a4570, 0xc1debe4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1d8a901e0, 0xc0b275ba78, 0x9, 0x9, 0xc196e8cae0, 0xc227e27df7, 0xc227e27d80) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1d8a901e0, 0xc0b275ba78, 0x9, 0x9, 0x9, 0x406594, 0xc173cee660, 0xc227e27df7) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0b275ba78, 0x9, 0x9, 0x2b41140, 0xc1d8a901e0, 0x0, 0xc200000000, 0x2124f40, 0xc1aa1bb680) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0b275ba40, 0xc1aa1bb660, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc136f0c900, 0xc290853aa0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc136f0c900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc136f0c900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4582672 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0fb8f86c0, 0xc1e07df000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc000ce5540, 0xc05bf41200, 0xc28e8c06c0, 0x40f967, 0xc000069538) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc000ce5540, 0xc05bf41200, 0xc000ce5540, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc05bf41200, 0x2b485c0, 0xc000ce5540, 0x0, 0x0, 0x0, 0xc9e3ffeec8, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000bfaba0, 0xc05bf41200, 0x0, 0x0, 0x0, 0xc9e3ffeec8, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000bfaba0, 0xc05bf41200, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc07f961a40, 0xc07f961b00, 0xc07f961bc0, 0xc00926ed90, 0xc225a73c80, 0xc044daa9a0, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4514692 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f63c0239548, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc238281998, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc238281980, 0xc22f1f6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc238281980, 0xc22f1f6000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000067) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0060fa428, 0xc22f1f6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc23b73b680, 0xc0460c2d58, 0x9, 0x9, 0xc2293b7d40, 0x457243, 0xb5b9d887065e) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc23b73b680, 0xc0460c2d58, 0x9, 0x9, 0x9, 0xbffdee9a6df86104, 0x15c1621decc, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0460c2d58, 0x9, 0x9, 0x2b41140, 0xc23b73b680, 0x0, 0x0, 0xc25e600ce0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c2d20, 0xc25e600ce0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc21d477500, 0xc22f1da870, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc21d477500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc21d477500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4582522 [select]: | |
| net/http.(*persistConn).readLoop(0xc9aa80f7a0) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4561338 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc136f0c900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4514795 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f58853b1910, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc238281b98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc238281b80, 0xc233578000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc238281b80, 0xc233578000, 0x8000, 0x8000, 0x203067, 0x420bb5, 0xc19c662500) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0060fa458, 0xc233578000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc0c0910420, 0xc0460c3298, 0x9, 0x9, 0xc0072d2e00, 0x1007f63e7434a30, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc0c0910420, 0xc0460c3298, 0x9, 0x9, 0x9, 0x601742e9, 0xc0594e9dc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0460c3298, 0x9, 0x9, 0x2b41140, 0xc0c0910420, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c3260, 0xc25cb2f500, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1a3430180, 0xc22f1db830, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1a3430180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1a3430180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4084495 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc2230af680, 0xc23e980000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e8fa660, 0xc23e980000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc225a73bf0, 0xc23e980000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc02ebddc70, 0x7f59e0599060, 0xc225a73bf0, 0xc23e980000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc02ebddc70, 0x7f59e0599060, 0xc225a73bf0, 0xc23e980000, 0x8000, 0x8000, 0x2b6fc80, 0xc225a73bf0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc00320edc0, 0x2b44ea0, 0xc02ebddc70, 0xc23e980000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xfe73c0b287ee7601, 0xfb22083c60000713, 0xc11632f3c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xfe73c0b287ee7601, 0xfb22083c60000713, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xfe73c0b287ee7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc007031700, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4582293 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f5886377210, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1e8864d18, 0x72, 0xc900, 0xc92b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1e8864d00, 0xc242c92000, 0xc92b, 0xc92b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1e8864d00, 0xc242c92000, 0xc92b, 0xc92b, 0x203095, 0x420bb5, 0xc255812920) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1c67f5108, 0xc242c92000, 0xc92b, 0xc92b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc255812920, 0xc242c92000, 0xc92b, 0xc92b, 0x410, 0x44c7, 0xc24645ee90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc07a983ad8, 0x2b41380, 0xc255812920, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc07a983880, 0x2b48520, 0xc1c67f5108, 0x5, 0xc1c67f5108, 0x400) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc07a983880, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc07a983880, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0f7ded7a0, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x400, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc23e379380, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x400, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc23c57ba80, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x400, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc997c7fac0, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x400, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc997c7fac0, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc997c7fb00, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc997c7fb00, 0xc2270da000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1db5cf1a0, 0xc1db5cf200, 0xc1db5cf260, 0xc0c5f020b0, 0xc1a1b7c570, 0xc13f938160, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4582751 [select]: | |
| net/http.(*persistConn).writeLoop(0xc10609c000) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4582743 [select]: | |
| net/http.(*persistConn).writeLoop(0xc24422a900) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4556671 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc183057e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4463649 [semacquire, 3 minutes]: | |
| sync.runtime_Semacquire(0xc216327f90) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc216327f88) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc216327f80, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc23b0830b0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc216327ec0, 0x255e040, 0x18, 0xc1e5d6d3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc22276ec00, 0x2bbccc0, 0xc150601c00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc22276ec00, 0x7f63bf19f960, 0xc1fb5f97c0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc22276ec00, 0x2bbbd60, 0xc1fb5f97c0, 0xc000863100, 0xc0fe16d688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc150601b60, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc150601b40, 0xc1506019e0, 0x26b3908, 0xc150601b40, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc150601b40, 0x2b99d40, 0xc216327d70) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc150601b20, 0xc1506019e0, 0xc150601a00, 0x7f63e743af10, 0xc10000cd80) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc150601b20, 0xc1506019e0, 0xc150601a00, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc150601b20, 0x2b99d40, 0xc216327d40) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0xc1506019e0, 0xc150601a20, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0x1, 0xc1506019a0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0xc1506019e0, 0xc150601a40, 0x6517b34, 0xb57ad39b9bad) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0x203054, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0xc1506019e0, 0xc150601a60, 0xc1506019e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0xc102211c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0xc1506019e0, 0x26b3908, 0x2b99d40, 0xc216327ce0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc071fc8780, 0xc1efed1a00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc071fc8780, 0xc1efed1a00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc075a62c50, 0xc0002c3380, 0x2bc6d40, 0xc071fc8780, 0xc1efed1a00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4561273 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59decac5a0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc2900c3098, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc2900c3080, 0xc13121c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc2900c3080, 0xc13121c000, 0x1000, 0x1000, 0x4378cc, 0xc15e695b58, 0x4647d0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc007b7e060, 0xc13121c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| net/http.(*persistConn).Read(0xc27abf85a0, 0xc13121c000, 0x1000, 0x1000, 0xc176cdc300, 0xc15e695c58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc1ee584c60) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1ee584c60, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc169b849c0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc27abf85a0) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4582100 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5886af6a40, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc19c7d8f98, 0x72, 0xd700, 0xd761, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc19c7d8f80, 0xc2520a8000, 0xd761, 0xd761, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc19c7d8f80, 0xc2520a8000, 0xd761, 0xd761, 0x203095, 0x420bb5, 0xc2552394e0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc9cbac3438, 0xc2520a8000, 0xd761, 0xd761, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc2552394e0, 0xc2520a8000, 0xd761, 0xd761, 0x2070, 0x4ee0, 0xc184f9ae90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc07aca3758, 0x2b41380, 0xc2552394e0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc07aca3500, 0x2b48520, 0xc9cbac3438, 0x5, 0xc9cbac3438, 0x2060) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc07aca3500, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc07aca3500, 0xc23e98089a, 0x7766, 0x7766, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc22ca94a20, 0xc23e98089a, 0x7766, 0x7766, 0x89a, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc23e27ee40, 0xc23e98089a, 0x7766, 0x7766, 0x89a, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc250f160c0, 0xc23e98089a, 0x7766, 0x7766, 0x89a, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc003233400, 0xc23e98089a, 0x7766, 0x7766, 0x89a, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc003233400, 0xc23e98089a, 0x7766, 0x7766, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc003233440, 0xc23e98089a, 0x7766, 0x7766, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc003233440, 0xc23e980000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1e1e756e0, 0xc1e1e75740, 0xc1e1e757a0, 0xc16fc4fd10, 0xc1970f7e00, 0xc083acd550, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4463565 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c51d1e0, 0xc0eba24bf8, 0x8, 0xc1a3b90f00, 0x28, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1ee227d00, 0xc0ccb232a0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1ee227d00, 0xc151b900e8, 0x1, 0x1, 0xc1be9f2950, 0xd, 0x1, 0xc00832e380, 0x7f63e743ca30) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc1ee227d00, 0xc1ee23dbc0, 0xc151b900e8, 0x1, 0x1, 0xc1ee23d4a0, 0x2b44f80, 0xc237c704e0, 0xc000ce6120, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc0874b4ac0, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc237c704b0, 0xc007134d90) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4582730 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0fc7806c0, 0xc0052cf040, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc000ce5540, 0xc0614d7600, 0xc1ef8026c0, 0x40f967, 0xc000142e58) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc000ce5540, 0xc0614d7600, 0xc000ce5540, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0614d7600, 0x2b485c0, 0xc000ce5540, 0x0, 0x0, 0x0, 0xc0071fa9f0, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000bfaba0, 0xc0614d7600, 0x0, 0x0, 0x0, 0xc0071fa9f0, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000bfaba0, 0xc0614d7600, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0cc8dcea0, 0xc0cc8dcf00, 0xc0cc8dd0e0, 0xc1d3fe4c40, 0xc24bf2e1b0, 0xc0a306e420, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4582710 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f63c032e7c0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1a0ead718, 0x72, 0x1000, 0x109b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1a0ead700, 0xc14f802000, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1a0ead700, 0xc14f802000, 0x109b, 0x109b, 0x203094, 0x420bb5, 0xc2517d6700) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc150ab4cd0, 0xc14f802000, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc2517d6700, 0xc14f802000, 0x109b, 0x109b, 0x0, 0x41310b, 0xc153e88900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc07a506cd8, 0x2b41380, 0xc2517d6700, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc07a506a80, 0x2b48520, 0xc150ab4cd0, 0x5, 0xc150ab4cd0, 0x463b40) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc07a506a80, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc07a506a80, 0xc19df3f000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0f41e47e0, 0xc19df3f000, 0x1000, 0x1000, 0xc06ed7aa80, 0xc153e88c58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc1561881e0) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1561881e0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc25d3c2120) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0f41e47e0) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4562107 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5886856eb8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc28f145698, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc28f145680, 0xc240ed7000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc28f145680, 0xc240ed7000, 0x1000, 0x1000, 0x4db33a, 0xb5be9e8c5b7d, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc007b7e5c0, 0xc240ed7000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| net/http.(*connReader).Read(0xc272ee0900, 0xc240ed7000, 0x1000, 0x1000, 0xbffdeeb69dc65277, 0x1765542fa10, 0x3f6bae0) | |
| /usr/local/go/src/net/http/server.go:797 +0xf4 | |
| bufio.(*Reader).fill(0xc2234f2cc0) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc2234f2cc0, 0x4, 0x1765542fa10, 0x3f6bae0, 0x0, 0x0, 0x3f6bae0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*conn).serve(0xc285589360, 0x2b99c80, 0xc00d5b9980) | |
| /usr/local/go/src/net/http/server.go:1949 +0x9d7 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:2962 +0x35c | |
| goroutine 4582520 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5a90be4008, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc23ac1a898, 0x72, 0xfb00, 0xfb29, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc23ac1a880, 0xc25b894000, 0xfb29, 0xfb29, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc23ac1a880, 0xc25b894000, 0xfb29, 0xfb29, 0x203095, 0x420bb5, 0xc256c4d260) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xca27c505d0, 0xc25b894000, 0xfb29, 0xfb29, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc256c4d260, 0xc25b894000, 0xfb29, 0xfb29, 0x0, 0x41310b, 0xc05545b900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc073b25058, 0x2b41380, 0xc256c4d260, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc073b24e00, 0x2b48520, 0xca27c505d0, 0x5, 0xca27c505d0, 0x40c924) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc073b24e00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc073b24e00, 0xc23b142000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0fd6c8000, 0xc23b142000, 0x1000, 0x1000, 0xc0cc95a720, 0xc05545bc58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc147ddb560) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc147ddb560, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0167b3560) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0fd6c8000) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4464097 [semacquire, 4 minutes]: | |
| sync.runtime_Semacquire(0xc238b3e430) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc238b3e428) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc238b3e420, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc238b582d0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc238b3e360, 0x255e040, 0x18, 0xc1ea8313f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc23856be00, 0x2bbccc0, 0xc0ccb1ba80, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc23856be00, 0x7f63bf19f960, 0xc238a7a690, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc23856be00, 0x2bbbd60, 0xc238a7a690, 0xc000863100, 0xc0062ed688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc0ccb1b9e0, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0ccb1b9c0, 0xc0ccb1b860, 0x26b3908, 0xc0ccb1b9c0, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0ccb1b9c0, 0x2b99d40, 0xc238b3e210) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0ccb1b9a0, 0xc0ccb1b860, 0xc0ccb1b880, 0x7f63e7430a30, 0xc040da4180) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0ccb1b9a0, 0xc0ccb1b860, 0xc0ccb1b880, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0ccb1b9a0, 0x2b99d40, 0xc238b3e1e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0xc0ccb1b860, 0xc0ccb1b8a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0x1, 0xc0ccb1b820) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0xc0ccb1b860, 0xc0ccb1b8c0, 0x3b240f32, 0xb57b086e2fab) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0x203033, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0xc0ccb1b860, 0xc0ccb1b8e0, 0xc0ccb1b860, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0xc2356cac68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0xc0ccb1b860, 0x26b3908, 0x2b99d40, 0xc238b3e180) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc023e1ac00, 0xc23857f000, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc023e1ac00, 0xc23857f000, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc03f97d970, 0xc0002c3380, 0x2bc6d40, 0xc023e1ac00, 0xc23857f000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4582742 [select]: | |
| net/http.(*persistConn).readLoop(0xc24422a900) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4561861 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0ab2bbe00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4581972 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00ddea134, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00ddea0b0, 0xc0d278c2c8, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294 | |
| github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc0db4967d0, 0xc114311bf0, 0x253e210, 0xb, 0xc1e914a0e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1693 +0x6f0 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc99c97fdc0, 0xc07df928b0, 0x2, 0x2, 0xc1683f3120, 0xd, 0x1, 0xc222d2d500, 0x7f63e742a180) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7 | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00618cb10, 0xc99c97fdc0, 0xc121ce5f80, 0xc07df928b0, 0x2, 0x2, 0xc121ce5920, 0x2b44f80, 0xc114311cb0, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc114311c80, 0xc1ea0c61c0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4498125 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).RLock(...) | |
| /usr/local/go/src/sync/rwmutex.go:50 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00c51d1e0, 0xc18980ebfd, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294 | |
| github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc2442bb7d0, 0xc122210780, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1713 +0x403 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc23ea43ec0, 0xc23887b2f0, 0x2, 0x2, 0xc0fb0fa610, 0xd, 0x1, 0xc005ff4000, 0x7f63e740ae98) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7 | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc23ea43ec0, 0xc1d675ecc0, 0xc23887b2f0, 0x2, 0x2, 0xc1549fe360, 0x2b44f80, 0xc1222108d0, 0xc00017c0b8, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1222108a0, 0xc233170380) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4513293 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc065466f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4474341 [select, 3 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1918b0600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582731 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc0cc8dd140, 0xa, 0x2b99c80, 0xc0052cef00, 0xc24bf2e240) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 4514540 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc062f5e320, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc12b314900, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc054d93b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4510178 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc19966b800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4578950 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59deb9daa0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc53c8d2318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc53c8d2300, 0xc1ac984000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc53c8d2300, 0xc1ac984000, 0x8000, 0x8000, 0x203079, 0x420bb5, 0xc1e7194c20) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc00450cda0, 0xc1ac984000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1ff86f1a0, 0xc125fe7d18, 0x9, 0x9, 0xc00596ce00, 0x1007f63e741ef10, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1ff86f1a0, 0xc125fe7d18, 0x9, 0x9, 0x9, 0x601742e9, 0xc036818dc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc125fe7d18, 0x9, 0x9, 0x2b41140, 0xc1ff86f1a0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc125fe7ce0, 0xc258491de0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1e65f3800, 0xc19a278840, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1e65f3800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1e65f3800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4515934 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00bc80f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4515945 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc028eec180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4577951 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0c41d3800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4513604 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc10a44af00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4515928 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc07661ac30, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19149ea80, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00bc80d80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4388078 [semacquire, 4 minutes]: | |
| sync.runtime_Semacquire(0xc0f4f5bfc0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0f4f5bfb8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0f4f5bfb0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc0ff209d40) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc0f4f5bef0, 0x255e040, 0x18, 0xc0f55533f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1c9270d80, 0x2bbccc0, 0xc088f91540, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1c9270d80, 0x7f63bf19f960, 0xc0fb69f2d0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1c9270d80, 0x2bbbd60, 0xc0fb69f2d0, 0xc000863100, 0xc005b4d688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc088f914a0, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc088f91480, 0xc088f91320, 0x26b3908, 0xc088f91480, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc088f91480, 0x2b99d40, 0xc0f4f5bda0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc088f91460, 0xc088f91320, 0xc088f91340, 0x7f63e740c2f0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc088f91460, 0xc088f91320, 0xc088f91340, 0x2b99d01, 0xc088f91460) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc088f91460, 0x2b99d40, 0xc0f4f5bd70) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0xc088f91320, 0xc088f91360, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0x1, 0xc088f912e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0xc088f91320, 0xc088f91380, 0x1d73b811, 0xb579c0b7e68f) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0x203022, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x40c700, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0xc088f91320, 0xc088f913a0, 0xc088f91320, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0xc11e6edc68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0xc088f91320, 0x26b3908, 0x2b99d40, 0xc0f4f5bd10) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc04ffaa780, 0xc0fb6a6a00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc04ffaa780, 0xc0fb6a6a00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0ca383b40, 0xc0002c3380, 0x2bc6d40, 0xc04ffaa780, 0xc0fb6a6a00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4517272 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f58871547d0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc06ce7e398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc06ce7e380, 0xc1495b2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc06ce7e380, 0xc1495b2000, 0x8000, 0x8000, 0x203086, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc143d460a8, 0xc1495b2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1f71baba0, 0xc0a65ff538, 0x9, 0x9, 0xc1ee092a80, 0x1007f63e7424368, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1f71baba0, 0xc0a65ff538, 0x9, 0x9, 0x9, 0x601742e2, 0xc071598dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0a65ff538, 0x9, 0x9, 0x2b41140, 0xc1f71baba0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0a65ff500, 0xc0a8b0c0a0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc003a9ea80, 0xc246e60a50, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc003a9ea80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc003a9ea80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4556749 [running]: | |
| goroutine running on other thread; stack unavailable | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4582071 [semacquire]: | |
| sync.runtime_Semacquire(0xc1e8866760) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1e8866758) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1e8866750, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc19d4c7fb0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1e8866660, 0x255e040, 0x18, 0xc16079d3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc197b33380, 0x2bbccc0, 0xc1e886a600, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc197b33380, 0x7f63bf19f960, 0xc009911860, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc197b33380, 0x2bbbd60, 0xc009911860, 0xc000863100, 0xc2400cd688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1e886a560, 0x24321a0, 0xc00046a000) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e886a540, 0xc1e886a3e0, 0x26b3908, 0xc1e886a540, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e886a540, 0x2b99d40, 0xc1e8866510) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e886a520, 0xc1e886a3e0, 0xc1e886a400, 0x7f63e742f5d8, 0xc1eb50be00) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e886a520, 0xc1e886a3e0, 0xc1e886a400, 0x2ade86a, 0x0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e886a520, 0x2b99d40, 0xc1e88664e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0xc1e886a3e0, 0xc1e886a420, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0x1, 0xc1e886a3a0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0xc1e886a3e0, 0xc1e886a440, 0xa9fde99, 0xb5ba67fb6914) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0x20307a, 0x420bb5) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0xc1e886a3e0, 0xc1e886a460, 0xc1e886a3e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0xc0005ecc68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0xc1e886a3e0, 0x26b3908, 0x2b99d40, 0xc1e8866480) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc1038bcf00, 0xc0e250f300, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc1038bcf00, 0xc0e250f300, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc10888b490, 0xc0002c3380, 0x2bc6d40, 0xc1038bcf00, 0xc0e250f300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4486230 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0e90c9540, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc12843aea0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1c1c82180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4511257 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f59dae18fc8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1951f1918, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1951f1900, 0xc1ffc60000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1951f1900, 0xc1ffc60000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000073) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc9e49f4068, 0xc1ffc60000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1d891e540, 0xc0a63f4f18, 0x9, 0x9, 0xc227e09d40, 0x457243, 0xb5b93aff39c9) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1d891e540, 0xc0a63f4f18, 0x9, 0x9, 0x9, 0xbffdee99c7a47217, 0x15b78985b71, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0a63f4f18, 0x9, 0x9, 0x2b41140, 0xc1d891e540, 0x0, 0x0, 0xc1fe98ae60, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0a63f4ee0, 0xc1fe98ae60, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1245db680, 0xc10e4704e0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1245db680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1245db680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4507827 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5889b51af0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc07c6dea98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc07c6dea80, 0xc14940f000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc07c6dea80, 0xc14940f000, 0x1000, 0x1000, 0x4db33a, 0xb5bf2411151e, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc000714048, 0xc14940f000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| net/http.(*connReader).Read(0xc003542180, 0xc14940f000, 0x1000, 0x1000, 0xbffdeeb78bcb3a74, 0x17731b309f5, 0x3f6bae0) | |
| /usr/local/go/src/net/http/server.go:797 +0xf4 | |
| bufio.(*Reader).fill(0xc1d73b00c0) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1d73b00c0, 0x4, 0x17731b309f5, 0x3f6bae0, 0x0, 0x0, 0x3f6bae0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*conn).serve(0xc0acf443c0, 0x2b99c80, 0xc233135f00) | |
| /usr/local/go/src/net/http/server.go:1949 +0x9d7 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:2962 +0x35c | |
| goroutine 4578955 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0fe80c140, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19958b140, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc079945980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4517271 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc003a9ea80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4582113 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f63c07862c0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1fffd2398, 0x72, 0xba00, 0xba45, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1fffd2380, 0xc20a024000, 0xba45, 0xba45, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1fffd2380, 0xc20a024000, 0xba45, 0xba45, 0x203095, 0x420bb5, 0xc256620180) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc00a33bfd0, 0xc20a024000, 0xba45, 0xba45, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc256620180, 0xc20a024000, 0xba45, 0xba45, 0x0, 0x41310b, 0xc13be0b900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc22c4ed758, 0x2b41380, 0xc256620180, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc22c4ed500, 0x2b48520, 0xc00a33bfd0, 0x5, 0xc00a33bfd0, 0xc08f6a2900) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc22c4ed500, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc22c4ed500, 0xc2007ca000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc2433439e0, 0xc2007ca000, 0x1000, 0x1000, 0xc1647adce0, 0xc13be0bc58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc1977abce0) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1977abce0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0b5fbbb60) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc2433439e0) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4514699 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc21d477b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment