Created
August 17, 2021 14:58
-
-
Save tbg/0266e0be68d1bc696925a544224000c4 to your computer and use it in GitHub Desktop.
This file has been truncated, but you can view the full file.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| === RUN TestKVNemesisMultiNode | |
| test_log_scope.go:73: test logs captured to: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestKVNemesisMultiNode079909290 | |
| test_log_scope.go:74: use -show-logs to present logs inline | |
| panic: test timed out after 40m0s | |
| goroutine 829917 [running]: | |
| testing.(*M).startAlarm.func1() | |
| /usr/local/go/src/testing/testing.go:1700 +0xe5 | |
| created by time.goFunc | |
| /usr/local/go/src/time/sleep.go:180 +0x45 | |
| goroutine 1 [chan receive, 39 minutes]: | |
| testing.(*T).Run(0xc000603c80, 0x4388341, 0x16, 0x49a9730, 0x4b3101) | |
| /usr/local/go/src/testing/testing.go:1239 +0x2da | |
| testing.runTests.func1(0xc000103200) | |
| /usr/local/go/src/testing/testing.go:1511 +0x78 | |
| testing.tRunner(0xc000103200, 0xc000b7fd70) | |
| /usr/local/go/src/testing/testing.go:1193 +0xef | |
| testing.runTests(0xc0005c7680, 0x76764a0, 0x8, 0x8, 0xc03ea1388f2469e2, 0x22ece768739, 0x79b3520, 0xc000bcc2c0) | |
| /usr/local/go/src/testing/testing.go:1509 +0x2fe | |
| testing.(*M).Run(0xc0005d7a80, 0x0) | |
| /usr/local/go/src/testing/testing.go:1417 +0x1eb | |
| github.com/cockroachdb/cockroach/pkg/kv/kvnemesis.TestMain(0xc0005d7a80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvnemesis/main_test.go:30 +0xbe | |
| main.main() | |
| _testmain.go:59 +0x165 | |
| goroutine 3642 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00049e150) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00049e150, 0xc0003a22a0, 0x1, 0x1, 0xc001f0dfb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001c99700) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 8 [chan receive, 40 minutes]: | |
| github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc00025f450) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149 | |
| created by github.com/klauspost/compress/zstd.newBlockDec | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173 | |
| goroutine 9 [chan receive, 40 minutes]: | |
| github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc00025f520) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149 | |
| created by github.com/klauspost/compress/zstd.newBlockDec | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173 | |
| goroutine 10 [chan receive, 40 minutes]: | |
| github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc00025f5f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149 | |
| created by github.com/klauspost/compress/zstd.newBlockDec | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173 | |
| goroutine 11 [chan receive, 40 minutes]: | |
| github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc00025f6c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149 | |
| created by github.com/klauspost/compress/zstd.newBlockDec | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173 | |
| goroutine 12 [chan receive, 40 minutes]: | |
| github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc00025f790) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149 | |
| created by github.com/klauspost/compress/zstd.newBlockDec | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173 | |
| goroutine 13 [chan receive, 40 minutes]: | |
| github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc00025f860) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149 | |
| created by github.com/klauspost/compress/zstd.newBlockDec | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173 | |
| goroutine 14 [chan receive, 40 minutes]: | |
| github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc00025f930) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149 | |
| created by github.com/klauspost/compress/zstd.newBlockDec | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173 | |
| goroutine 15 [chan receive, 40 minutes]: | |
| github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc00025fa00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:215 +0x149 | |
| created by github.com/klauspost/compress/zstd.newBlockDec | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/klauspost/compress/zstd/blockdec.go:118 +0x173 | |
| goroutine 16 [chan receive]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon() | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x74 | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.init.6 | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x35 | |
| goroutine 18 [chan receive, 40 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher() | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0x12c | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.init.6 | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x4d | |
| goroutine 50 [syscall, 40 minutes]: | |
| os/signal.signal_recv(0x0) | |
| /usr/local/go/src/runtime/sigqueue.go:168 +0xa5 | |
| os/signal.loop() | |
| /usr/local/go/src/os/signal/signal_unix.go:23 +0x25 | |
| created by os/signal.Notify.func1.1 | |
| /usr/local/go/src/os/signal/signal.go:151 +0x45 | |
| goroutine 3577 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00132a180) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00132a180, 0xc0003a22a0, 0x1, 0x1, 0xc001c607b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000510e00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 70 [chan receive]: | |
| github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1() | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:154 +0x189 | |
| created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0 | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:142 +0x35 | |
| goroutine 3522 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc003786b80) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc003786b78) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc003786b70, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc003786b70, 0x5922350, 0xc006aa1a80, 0xc004e81728, 0xc006aa1a80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GroupWorkers(0x5922388, 0xc000074108, 0x5, 0xc00220dc20, 0x4, 0x4) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:177 +0x111 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvnemesis.RunNemesis(0x5922388, 0xc000074108, 0xc000abc030, 0xc002f311a0, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvnemesis/kvnemesis.go:82 +0x3a5 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvnemesis.TestKVNemesisMultiNode(0xc000603c80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvnemesis/kvnemesis_test.go:107 +0x548 | |
| testing.tRunner(0xc000603c80, 0x49a9730) | |
| /usr/local/go/src/testing/testing.go:1193 +0xef | |
| created by testing.(*T).Run | |
| /usr/local/go/src/testing/testing.go:1238 +0x2b3 | |
| goroutine 3567 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eab80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3519 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| google.golang.org/grpc.(*Server).Serve(0xc0029bc1c0, 0x5905590, 0xc0030b42b8, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:786 +0x27f | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2232 +0x3f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0032b6000) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5707 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc00015ca80, 0x10, 0xc0025314a0, 0x596be38, 0xc006480ec0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:257 +0x670 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:143 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc00250e800, 0xc0025314a0, 0x596be38, 0xc006480ec0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2530 +0x136 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc001e268c0, 0xc0025314a0, 0x596be38, 0xc006480ec0, 0x10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:216 +0xce | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc00283cc00, 0xc0025314a0, 0x596be38, 0xc006480ec0, 0xc00283cc00, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:1101 +0x54 | |
| github.com/cockroachdb/cockroach/pkg/roachpb._Internal_RangeFeed_Handler(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0022d43c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8526 +0x113 | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0022d43c0, 0xc005f361f8, 0x49aa000, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:170 +0x5f7 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0022d43c0, 0x40c4c00, 0xc0064e8a40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0022d43c0, 0xc005f361f8, 0xc0064e8a40, 0xc0064e8a40, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:186 +0x99 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0022d43c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x1, 0x42ead40, 0xc00283cc00, 0x5966670, 0xc0022d43c0, 0xc005f361f8, 0xc0064e8a00, 0x40c4c00, 0xc0064e8a00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb7 | |
| google.golang.org/grpc.chainStreamServerInterceptors.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0022d43c0, 0xc005f361f8, 0x49aa000, 0x59223f8, 0xc0064f4fc0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1401 +0xbd | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0029bc1c0, 0x5981b58, 0xc00219f980, 0xc002df9320, 0xc0015df830, 0x765a660, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1549 +0x6d8 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0029bc1c0, 0x5981b58, 0xc00219f980, 0xc002df9320, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1622 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00610ddf0, 0xc0029bc1c0, 0x5981b58, 0xc00219f980, 0xc002df9320) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:941 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:939 +0x1fd | |
| goroutine 4877 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:113 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc0024892a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5600 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001466320, 0xc005f95270, 0x5, 0x5, 0xffc03c, 0xc00a7121b0, 0xc02780200c) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001466320, 0xc005f95270, 0x5, 0x5, 0xc0061872c0, 0xc003d64678, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0065c2f00, 0xc005f95270, 0x5, 0x5, 0x0, 0xc006187320, 0xacfcdc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc0065c2f00, 0xc005f95270, 0x5, 0x5, 0x5, 0xc006187390, 0x426f05, 0x3da3b20) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc005c59440, 0xc005f95270, 0x5, 0x5, 0xc027802000, 0x16, 0x200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc005f95260, 0x7fffffff, 0x19e2de2, 0x7a02de8, 0xc027802000, 0x8, 0x200, 0x4252220) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc005f95260, 0xc005c59440, 0x0, 0x0, 0x7fffffff, 0xc006187640, 0x59058c0, 0x7a02de8, 0xc0278001e0, 0x5a3df06e745, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc005f95260, 0x7f06bca6db20, 0xc00044af90, 0xc005c59440, 0x0, 0x0, 0x4252220, 0xc012ace6f0, 0x7fffffff, 0xc006187640, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc000b49a00, 0x4252220, 0xc012ace6f0, 0xc006187640, 0x45663c, 0x770000c0061876a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:957 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000b49a00, 0xc0061876e8, 0x421105) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:808 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc005c59200, 0xc0061877c0, 0xc006187790, 0x18, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:662 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc005c59200, 0x4252220, 0xc012ace6f0, 0x2, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:807 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc006346820, 0xc0061878c8, 0x0, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8375 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc000ebc000, 0x5922350, 0xc00203a300, 0xc0005c4510, 0x1, 0x8, 0xc0005c4518, 0x1, 0x1, 0x169bb531ad9aa2aa, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:270 +0x911 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc000ebc000, 0x5922350, 0xc00203a300, 0xc00376cfc0, 0x0, 0xc00065e360, 0xc0000300c0, 0xaaad59, 0xc0063591a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:151 +0x34c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1(0x5922350, 0xc00203a300, 0xc00484dfa8, 0xb2c26b) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:70 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0xc005f28900, 0xc005c590e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc00066fe90, 0xc0065c2d20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 5601 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc001a87880, 0xc005c59200, 0x59223f8, 0xc0065c2e40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 3644 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00049e210) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00049e210, 0xc0003a22a0, 0x1, 0x1, 0xc001f1bfb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001c99900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3568 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eab90) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5484 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:380 +0x5a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0000bdae0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4672 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00375e1c0, 0x5922350, 0xc003665840, 0xc003665840, 0xc002c42ec0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00375e1c0, 0x5922350, 0xc003665840, 0x10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:276 +0xd4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x5922350, 0xc003665740) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:263 +0x6d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc003665740, 0x0, 0x0, 0xc0036ff9b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5226 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc0036ac900, 0xc001376550, 0xc00369a620, 0xf, 0xc001a490e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc0026d5240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc0026d5240, 0x0, 0x0, 0xc001376690) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3517 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc003462000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 3630 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d89760) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3924 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc00093f720) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc00093f718) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc00093f710, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc00093f710, 0x5922350, 0xc000a769c0, 0xc0008f5080, 0xc000a769c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc001e7e000, 0x5922350, 0xc00093f410, 0xc000f17a90, 0x1, 0x8, 0xc000f17a98, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:83 +0x3ca | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc000640810, 0x5922350, 0xc002544400, 0xc000f17a90, 0x1, 0x8, 0xc000f17a98, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:69 +0xb4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1(0x5922350, 0xc002544400) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:220 +0xe3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc002544400, 0x0, 0x0, 0xc00093f3b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3677 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044b030) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3620 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d89660) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3413 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000bcae00, 0x5922350, 0xc001f02240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x8e | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/log/flags.go:277 +0xb66 | |
| goroutine 3616 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00132a3f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00132a3f0, 0xc0003a22a0, 0x1, 0x1, 0xc001f31fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124b800) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3678 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044b040) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3604 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.newInfoStore.func1(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/infostore.go:190 +0x14d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc000698940) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3603 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc000d48d80, 0x5922388, 0xc000074108, 0xc001b9d1d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:235 +0x127 | |
| created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler | |
| /go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:232 +0x65 | |
| goroutine 3602 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x5922350, 0xc00063f4c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:403 +0x51 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc00063f4c0, 0x0, 0x0, 0xc003761668) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5496 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x7) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31050) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3582 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00132a240) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00132a240, 0xc0003a22a0, 0x1, 0x1, 0xc0020e0fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124a100) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3584 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0009285e0, 0x7f0600000435) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0009285d0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193 | |
| github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc000928500, 0x59223f8, 0xc00132a300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b | |
| runtime/pprof.Do(0x5922388, 0xc00132a300, 0xc0003a2180, 0x1, 0x1, 0xc0020e07b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc000928500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89 | |
| created by github.com/cockroachdb/pebble/record.NewLogWriter | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea | |
| goroutine 4900 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc003b50900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5451 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| net/http.(*Server).Serve(0xc002465dc0, 0x5905590, 0xc00118dbd8, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2961 +0x285 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func3(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2296 +0x127 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc005ac3260) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3690 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc001b5f6c0, 0x5922350, 0xc0039ef6c0, 0xc0014259b0, 0xc00059f680, 0xc00059f800, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xe7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc001b5f6c0, 0x5922350, 0xc0039ef6c0, 0xc0014259b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:226 +0x5f8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*Factory).RangeFeed.func1(0x5922350, 0xc0039ef6c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:137 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc0039ef6c0, 0x0, 0x0, 0xc002c944c8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5280 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc00015ca80, 0x10, 0xc0012c4d20, 0x596be38, 0xc003b300a0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:257 +0x670 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:143 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc00250e800, 0xc0012c4d20, 0x596be38, 0xc003b300a0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2530 +0x136 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc001e268c0, 0xc0012c4d20, 0x596be38, 0xc003b300a0, 0x10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:216 +0xce | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc00283cc00, 0xc0012c4d20, 0x596be38, 0xc003b300a0, 0xc00283cc00, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:1101 +0x54 | |
| github.com/cockroachdb/cockroach/pkg/roachpb._Internal_RangeFeed_Handler(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004f7b200, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8526 +0x113 | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004f7b200, 0xc0026069d8, 0x49aa000, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:170 +0x5f7 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004f7b200, 0x40c4c00, 0xc001f02980) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004f7b200, 0xc0026069d8, 0xc001f02980, 0xc001f02980, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:186 +0x99 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004f7b200, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x1, 0x42ead40, 0xc00283cc00, 0x5966670, 0xc004f7b200, 0xc0026069d8, 0xc001f02900, 0x40c4c00, 0xc001f02900) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb7 | |
| google.golang.org/grpc.chainStreamServerInterceptors.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004f7b200, 0xc0026069d8, 0x49aa000, 0x59223f8, 0xc0056db8f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1401 +0xbd | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0029bc1c0, 0x5981b58, 0xc003cf3b00, 0xc00407d8c0, 0xc0015df830, 0x765a660, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1549 +0x6d8 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0029bc1c0, 0x5981b58, 0xc003cf3b00, 0xc00407d8c0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1622 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00456efc0, 0xc0029bc1c0, 0x5981b58, 0xc003cf3b00, 0xc00407d8c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:941 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:939 +0x1fd | |
| goroutine 3605 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc001c76f00, 0xc002642120) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3575 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.startPurgeOldSessions.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/purge_auth_session.go:65 +0x225 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc000479a20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3576 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00132a0f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00132a0f0, 0xc0003a22a0, 0x1, 0x1, 0xc001c60fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc003327e00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3518 [select]: | |
| net.(*pipe).read(0xc001936180, 0xc00343c000, 0x773, 0x773, 0xc00343c005, 0x12, 0x76e) | |
| /usr/local/go/src/net/pipe.go:159 +0x185 | |
| net.(*pipe).Read(0xc001936180, 0xc00343c000, 0x773, 0x773, 0x428e5b, 0xc0291bc7e0, 0x18) | |
| /usr/local/go/src/net/pipe.go:142 +0x56 | |
| crypto/tls.(*atLeastReader).Read(0xc0291bc7e0, 0xc00343c000, 0x773, 0x773, 0x76e, 0xc000382000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00267c278, 0x58a0580, 0xc0291bc7e0, 0x426be5, 0x3da8720, 0x4119f40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00267c000, 0x7f0691997fc0, 0xc001936180, 0x5, 0xc001936180, 0x11) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00267c000, 0x0, 0x0, 0xa9c5be) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00267c000, 0xc00344a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc002216de0, 0xc00342a2d8, 0x9, 0x9, 0x14, 0x7f06bcad8a68, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc002216de0, 0xc00342a2d8, 0x9, 0x9, 0x9, 0xa7fa45, 0xc00bfa49e4, 0xc00500ae60) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00342a2d8, 0x9, 0x9, 0x58a03a0, 0xc002216de0, 0x0, 0x0, 0xc00bfa49d8, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00342a2a0, 0xc00bfa49d8, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc003462000, 0xc001faa810, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0029bc1c0, 0x5981b58, 0xc003462000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0029bc1c0, 0x5981b58, 0xc003462000, 0x435fb83, 0x8) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 3578 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00132a1e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00132a1e0, 0xc0003a22a0, 0x1, 0x1, 0xc0020dffb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000510f00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3516 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001973180, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc003682000, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc003462000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 3515 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001973220, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002216f00, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc003464000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 4917 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0020fdc20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5459 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc0000e6380, 0xc005c64120, 0x59223f8, 0xc005c6a210) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 3574 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc000145450, 0x5922350, 0xc0014ffd40, 0xc0022a2db0, 0xc0002ba600, 0xc0002bbc80, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xe7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc000145450, 0x5922350, 0xc0014ffd40, 0xc0022a2db0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:226 +0x5f8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*Factory).RangeFeed.func1(0x5922350, 0xc0014ffd40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:137 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc0014ffd40, 0x0, 0x0, 0xc0037610f8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3736 [select, 4 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).run(0xc00211ad80, 0x59223f8, 0xc001ca68a0, 0xc000e3e380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:121 +0x296 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).Start.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:100 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0013147e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3906 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc00254d400) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5463 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc004dcde00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 3573 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc003761080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5452 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| crypto/tls.(*listener).Accept(0xc00118dc20, 0x40dbce0, 0xc0052972c0, 0x3d1ed60, 0x7646620) | |
| /usr/local/go/src/crypto/tls/tls.go:67 +0x37 | |
| net/http.(*Server).Serve(0xc001347880, 0x58ef210, 0xc00118dc20, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2961 +0x285 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func4(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2309 +0x3f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc00118dc38) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7277 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0054eebc0, 0x593db70, 0xc007496930) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:357 +0x15f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc00256de80, 0x593db70, 0xc007496930, 0x43d650a, 0x29, 0xc004d1bfb0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:307 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:312 +0x17e | |
| created by time.goFunc | |
| /usr/local/go/src/time/sleep.go:180 +0x45 | |
| goroutine 3740 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoop(0xc00211b440, 0x5922350, 0xc000a76400) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:214 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc000677390) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5504 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0xc000000011) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b310d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3661 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00011c450) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00011c450, 0xc0003a22a0, 0x1, 0x1, 0xc00202b7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001474700) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4874 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca2be30, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc002512798, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc002512780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc002512780, 0x18, 0xed8abfb01, 0x203000) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc000ef29a8, 0xc0016f2de0, 0x42f37d, 0xc00118a668) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc000ef29a8, 0x0, 0x0, 0x7f06919b6b58, 0x28) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc000aca838, 0x59223f8, 0xc0012ca8d0, 0xc000d48d80, 0x5905230, 0xc000ef29a8, 0xc00353eea0, 0xc00482a631, 0xc002488db0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:135 +0x50 | |
| github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2330 +0xfc | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc00261b240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3662 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00011c4b0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00011c4b0, 0xc0003a22a0, 0x1, 0x1, 0xc0009c4fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001474800) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3663 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00011c390) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00011c390, 0xc0003a22a0, 0x1, 0x1, 0xc0009c1fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001474900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3659 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc002a69ea0, 0xc00142b3e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3914 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func7(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:817 +0x1ef | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc00254d540) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4783 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x1b) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1420) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3583 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002640750) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002640750, 0xc0003a22a0, 0x1, 0x1, 0xc0020ddfb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124a200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3581 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0022062a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0022062a0, 0xc0003a22a0, 0x1, 0x1, 0xc0020df7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124a000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4801 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x2f) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1540) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5228 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc002fcd340) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5406 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc001c4b880, 0xc004c55b00, 0x59223f8, 0xc005c8a750) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 5473 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func1(0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:193 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc006012060) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4720 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc003b50300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5270 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc002290240, 0xc0012aad70, 0xc00011f060, 0xf, 0xc0023e2060, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc001f02580) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc001f02580, 0x0, 0x0, 0xc0012aaeb0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5070 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc002270000) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270000, 0x0, 0x0, 0xc0024a8940) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4097 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc003e185a0, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc003cf1230) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3550 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00132a150) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00132a150, 0xc0003a22a0, 0x1, 0x1, 0xc00128dfb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124ad00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4777 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xc) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf13c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5283 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc0025589c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4806 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x34) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1590) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4722 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| crypto/tls.(*listener).Accept(0xc000ef28b8, 0x40dbce0, 0xc003898ab0, 0x3d1ed60, 0x7646620) | |
| /usr/local/go/src/crypto/tls/tls.go:67 +0x37 | |
| net/http.(*Server).Serve(0xc003302620, 0x58ef210, 0xc000ef28b8, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2961 +0x285 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func4(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2309 +0x3f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc000ef28d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3653 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc00256de80, 0x5922388, 0xc000074108, 0xc00014b590) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:235 +0x127 | |
| created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler | |
| /go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:232 +0x65 | |
| goroutine 3658 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc002a69d60, 0xc00142b380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3760 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2261 +0x53 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0030b4180) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3625 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d896e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3623 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d896c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3564 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eab50) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3739 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage.(*Storage).deleteSessionsLoop(0xc0033a8c60, 0x59223f8, 0xc0032b47c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage/slstorage.go:317 +0x1fa | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc000677330) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3929 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc002304000, 0x59223f8, 0xc00093fe00, 0x6, 0xc003760a20, 0xc000e3e380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:242 +0x40c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1(0x59223f8, 0xc00093fe00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:197 +0x59 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc00093fe00, 0x0, 0x0, 0xc0034007e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3761 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca2c000, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0030a4398, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc0030a4380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc0030a4380, 0xc, 0x7f06bcad9878, 0x10) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc0030b4168, 0x4295b8, 0xc, 0x3ec7fe0) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc0030b4168, 0xc0004a4160, 0x1165, 0x6964270a322a0000, 0x7265646e65737473) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cmux.(*cMux).Serve(0xc002f57d80, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x96 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func2(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2278 +0x2f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0030b41c8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4183 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc0018b3600) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 3563 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eab40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5284 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc0025589e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3543 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc001c772c0, 0xc002643260) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 4285 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc002427600) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 4802 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x30) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1550) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3721 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc002f77ff0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3737 [select, 9 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startSystemLogsGC.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server_systemlog_gc.go:177 +0x545 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0013147f8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3421 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002640390) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002640390, 0xc0003a22a0, 0x1, 0x1, 0xc00165ffb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000b5bf00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 5209 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache.(*Cache).periodicallyRefreshProtectedtsCache(0xc0021c5600, 0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache/cache.go:160 +0x239 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00268e120) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3686 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002c94408) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4727 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc002393770) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 4774 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x10) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1390) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3638 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0036feae0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3680 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044b0d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3758 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1(0x49b2640, 0xc000d72410, 0xc000639730, 0x59223f8, 0xc001ca68a0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:247 +0xff | |
| created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:240 +0xd0 | |
| goroutine 3679 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044b0a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3666 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00011c600) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00011c600, 0xc0003a22a0, 0x1, 0x1, 0xc0009c17b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001474c00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3614 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0022064e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0022064e0, 0xc0003a22a0, 0x1, 0x1, 0xc001f2dfb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124b600) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3968 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca2c1d0, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc002512618, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc002512600, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc002512600, 0xc, 0x7f06bcad8a68, 0x10) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc000ef2840, 0x4295b8, 0xc, 0x3ec7fe0) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc000ef2840, 0xc003795630, 0x1, 0x0, 0x0) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cmux.(*cMux).Serve(0xc001f031c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x96 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func2(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2278 +0x2f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc000ef28a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5269 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc003dc5b80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5494 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0xc000000002) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31030) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3621 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d89670) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3618 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc00092f4e0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00092f4d0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193 | |
| github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc00092f400, 0x59223f8, 0xc0026407e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b | |
| runtime/pprof.Do(0x5922388, 0xc0026407e0, 0xc0003a2180, 0x1, 0x1, 0xc001f0b7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc00092f400) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89 | |
| created by github.com/cockroachdb/pebble/record.NewLogWriter | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea | |
| goroutine 3673 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044afd0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3675 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044b000) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5073 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc002270120) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270120, 0x0, 0x0, 0xc0024a89a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5471 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc005c64c60, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc003b30fb0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5111 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca77a58, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0023a6118, 0x72, 0xc100, 0xc1c1, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0023a6100, 0xc0057d0000, 0xc1c1, 0xc1c1, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0023a6100, 0xc0057d0000, 0xc1c1, 0xc1c1, 0xc1bc, 0xc0057d0000, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc0003a4170, 0xc0057d0000, 0xc1c1, 0xc1c1, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc00ddf7f38, 0xc0057d0000, 0xc1c1, 0xc1c1, 0x5167a5, 0xc000280400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00250d778, 0x58a0580, 0xc00ddf7f38, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00250d500, 0x58a88a0, 0xc0003a4170, 0x5, 0xc0003a4170, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00250d500, 0x0, 0x0, 0xc029818690) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00250d500, 0xc004150000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0050fdda0, 0xc00342a578, 0x9, 0x9, 0x7f068b6b18b8, 0x48, 0xc02916e958) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0050fdda0, 0xc00342a578, 0x9, 0x9, 0x9, 0xc029190001, 0xc002db3d38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00342a578, 0x9, 0x9, 0x58a03a0, 0xc0050fdda0, 0x0, 0xc000000000, 0xaba293, 0xc004e04d20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00342a540, 0xc0291e7eb0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc004e04d20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 3548 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002640600) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002640600, 0xc0003a22a0, 0x1, 0x1, 0xc00128cfb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124aa00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4807 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x35) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf15a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3681 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044bdd0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3645 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00049e0f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00049e0f0, 0xc0003a22a0, 0x1, 0x1, 0xc001f177b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001c99a00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 5083 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*jobScheduler).runDaemon.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/job_scheduler.go:357 +0x272 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc001306ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3754 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc002fb7f50) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3757 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.startPurgeOldSessions.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/purge_auth_session.go:65 +0x225 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc002fe5be0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3755 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc002fb7f68) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3683 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044bdf0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3756 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc0030a65b0, 0x5922350, 0xc002f57c00, 0xc00207b5f0, 0xc0000300c0, 0xc000030180, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xe7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc0030a65b0, 0x5922350, 0xc002f57c00, 0xc00207b5f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:226 +0x5f8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*Factory).RangeFeed.func1(0x5922350, 0xc002f57c00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:137 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc002f57c00, 0x0, 0x0, 0xc002fb7fe0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3655 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc002a69ae0, 0xc00142b260) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 4913 [select, 9 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*TemporaryObjectCleaner).Start.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/temporary_schema.go:597 +0x2de | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc002fb6450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4687 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc003a35860) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 3674 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044afe0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4797 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x25) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1500) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3676 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044b010) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3628 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d89710) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5024 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:78 +0xc5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc0053b9e20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4184 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000aef440, 0xc0024055e0, 0xc001d726c0, 0xf, 0xc001abc960, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc00063f4c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc00063f4c0, 0x0, 0x0, 0xc002405720) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5468 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc004f7b680, 0x5922350, 0xc003c40200, 0xc002603520, 0x5978d58, 0xc003b31e90, 0xc002513080, 0xc00369bc80, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:371 +0x3bf | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1(0x5922350, 0xc003c40200) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:125 +0x44e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc003c40200, 0x0, 0x0, 0xc003c40240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4756 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1270) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3741 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier.(*Notifier).run(0xc000110d20, 0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier/notifier.go:136 +0xea | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0006773a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3878 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0014fd140) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd140, 0x0, 0x0, 0xc003300100) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3682 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044bde0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3668 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc0002085e0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0002085d0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193 | |
| github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc000208500, 0x59223f8, 0xc00011c660) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b | |
| runtime/pprof.Do(0x5922388, 0xc00011c660, 0xc0003a2180, 0x1, 0x1, 0xc00202dfb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc000208500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89 | |
| created by github.com/cockroachdb/pebble/record.NewLogWriter | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea | |
| goroutine 3589 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1(0x5922350, 0xc00203a300, 0xc002513080, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:66 +0x12f | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc00066fe90, 0xc00066ff20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4911 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoop(0xc0026328a0, 0x5922350, 0xc002ad7580) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:214 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc004db8230) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4908 [select, 9 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startSystemLogsGC.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server_systemlog_gc.go:177 +0x545 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc002fb62a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3724 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa020) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3726 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa040) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5276 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca77400, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00255ad18, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00255ad00, 0xc00559c000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00255ad00, 0xc00559c000, 0x8b5, 0x8b5, 0x8b0, 0xc00559c000, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00172c508, 0xc00559c000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc00c4caa98, 0xc00559c000, 0x8b5, 0x8b5, 0x5167a5, 0xc000aaa000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00062faf8, 0x58a0580, 0xc00c4caa98, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00062f880, 0x58a88a0, 0xc00172c508, 0x5, 0xc00172c508, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00062f880, 0x0, 0x0, 0xc0275066f0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00062f880, 0xc005a50000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc002557980, 0xc0023deb98, 0x9, 0x9, 0x7f068c553b20, 0x48, 0xc027849c18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc002557980, 0xc0023deb98, 0x9, 0x9, 0x9, 0xc023150001, 0xc0044cfd38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0023deb98, 0x9, 0x9, 0x58a03a0, 0xc002557980, 0x0, 0xc000000000, 0xaba293, 0xc003464f00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0023deb60, 0xc0275006e0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc003464f00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5421 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc00015ca80, 0x10, 0xc000d8f950, 0x596be38, 0xc002f77ec0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:257 +0x670 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:143 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc00250e800, 0xc000d8f950, 0x596be38, 0xc002f77ec0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2530 +0x136 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc001e268c0, 0xc000d8f950, 0x596be38, 0xc002f77ec0, 0x10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:216 +0xce | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc00283cc00, 0xc000d8f950, 0x596be38, 0xc002f77ec0, 0xc00283cc00, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:1101 +0x54 | |
| github.com/cockroachdb/cockroach/pkg/roachpb._Internal_RangeFeed_Handler(0x42ead40, 0xc00283cc00, 0x5966670, 0xc00221c3c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8526 +0x113 | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc00221c3c0, 0xc002e391a0, 0x49aa000, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:170 +0x5f7 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc00221c3c0, 0x40c4c00, 0xc003bc3500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x42ead40, 0xc00283cc00, 0x5966670, 0xc00221c3c0, 0xc002e391a0, 0xc003bc3500, 0xc003bc3500, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:186 +0x99 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc00221c3c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x1, 0x42ead40, 0xc00283cc00, 0x5966670, 0xc00221c3c0, 0xc002e391a0, 0xc003bc34c0, 0x40c4c00, 0xc003bc34c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb7 | |
| google.golang.org/grpc.chainStreamServerInterceptors.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc00221c3c0, 0xc002e391a0, 0x49aa000, 0x59223f8, 0xc005296d50) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1401 +0xbd | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0029bc1c0, 0x5981b58, 0xc0054e1c80, 0xc00581e7e0, 0xc0015df830, 0x765a660, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1549 +0x6d8 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0029bc1c0, 0x5981b58, 0xc0054e1c80, 0xc00581e7e0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1622 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0006fb8c0, 0xc0029bc1c0, 0x5981b58, 0xc0054e1c80, 0xc00581e7e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:941 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:939 +0x1fd | |
| goroutine 3725 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa030) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3727 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa050) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4187 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002405f90, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0046ff920, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc001ace780) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 4731 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca2ba90, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc002512718, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc002512700, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc002512700, 0x37e11d600, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc000ef2978, 0xc0036b1e00, 0xc0036b1e08, 0x28) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc000ef2978, 0x49a9320, 0xc001f03280, 0x597dab0, 0xc002c9b060) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cmux.(*cMux).Serve(0xc001f03280, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x96 | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.2.1() | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2237 +0x2f | |
| sync.(*Once).doSlow(0xc0036beb20, 0xc000ca4f08) | |
| /usr/local/go/src/sync/once.go:68 +0xec | |
| sync.(*Once).Do(...) | |
| /usr/local/go/src/sync/once.go:59 | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.2(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2236 +0x74 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0025852e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4805 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x2d) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1580) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5277 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0012ab950, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002557aa0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc003464f00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 3723 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa010) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3664 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00011c570) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00011c570, 0xc0003a22a0, 0x1, 0x1, 0xc0009bf7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001474a00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3693 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0012cb410) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0012cb410, 0xc0003a22a0, 0x1, 0x1, 0xc000fa37b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000574e00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3617 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002206540) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002206540, 0xc0003a22a0, 0x1, 0x1, 0xc001f2d7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124b900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3566 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eab70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3627 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d89700) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5324 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000f48550) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 4811 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x38) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf15e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3640 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc000649a00, 0x5922350, 0xc0026d4cc0, 0xc0022a3770, 0xc00059e600, 0xc00059e900, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xe7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc000649a00, 0x5922350, 0xc0026d4cc0, 0xc0022a3770) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:226 +0x5f8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*Factory).RangeFeed.func1(0x5922350, 0xc0026d4cc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:137 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc0026d4cc0, 0x0, 0x0, 0xc0036feb70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3420 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002640330) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002640330, 0xc0003a22a0, 0x1, 0x1, 0xc00165d7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000b5a800) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3579 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00132a060) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00132a060, 0xc0003a22a0, 0x1, 0x1, 0xc0020defb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000b5a000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 713667 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00092aae0, 0x1491) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00092aad0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193 | |
| github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc00092aa00, 0x59223f8, 0xc008f16030) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b | |
| runtime/pprof.Do(0x5922388, 0xc008f16030, 0xc0003a2180, 0x1, 0x1, 0xc0048517b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc00092aa00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89 | |
| created by github.com/cockroachdb/pebble/record.NewLogWriter | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea | |
| goroutine 4804 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x2e) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1570) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3656 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc002a69b80, 0xc00142b2c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 4284 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0024b3310) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 4760 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xa) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf12b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4179 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc003298780, 0x5978db0, 0xc000698fc0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:190 +0x6c5 | |
| github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x420a960, 0xc003298780, 0x5966670, 0xc001d70900, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:660 +0xad | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x420a960, 0xc003298780, 0x5966670, 0xc001d70900, 0xc002607128, 0x49a94f8, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:170 +0x5f7 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x420a960, 0xc003298780, 0x5966670, 0xc001d70900, 0x40c4c00, 0xc0009d2500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x420a960, 0xc003298780, 0x5966670, 0xc001d70900, 0xc002607128, 0xc0009d2500, 0xc0009d2500, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:186 +0x99 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x420a960, 0xc003298780, 0x5966670, 0xc001d70900, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x1, 0x420a960, 0xc003298780, 0x5966670, 0xc001d70900, 0xc002607128, 0xc0009d24c0, 0x40c4c00, 0xc0009d24c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb7 | |
| google.golang.org/grpc.chainStreamServerInterceptors.func1(0x420a960, 0xc003298780, 0x5966670, 0xc001d70900, 0xc002607128, 0x49a94f8, 0x59223f8, 0xc00397ed20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1401 +0xbd | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0029bc1c0, 0x5981b58, 0xc002a99200, 0xc001aa18c0, 0xc0015de750, 0x764e240, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1549 +0x6d8 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0029bc1c0, 0x5981b58, 0xc002a99200, 0xc001aa18c0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1622 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc004886fa0, 0xc0029bc1c0, 0x5981b58, 0xc002a99200, 0xc001aa18c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:941 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:939 +0x1fd | |
| goroutine 3419 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002640210) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002640210, 0xc0003a22a0, 0x1, 0x1, 0xc0016587b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000b5a700) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4721 [select]: | |
| net.(*pipe).read(0xc0014e5980, 0xc001a4b800, 0x773, 0x773, 0xc001a4b805, 0x12, 0x76e) | |
| /usr/local/go/src/net/pipe.go:159 +0x185 | |
| net.(*pipe).Read(0xc0014e5980, 0xc001a4b800, 0x773, 0x773, 0x428e5b, 0xc01727b7d0, 0x18) | |
| /usr/local/go/src/net/pipe.go:142 +0x56 | |
| crypto/tls.(*atLeastReader).Read(0xc01727b7d0, 0xc001a4b800, 0x773, 0x773, 0x76e, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00427e278, 0x58a0580, 0xc01727b7d0, 0x426be5, 0x3da8720, 0x4119f40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00427e000, 0x7f0691997fc0, 0xc0014e5980, 0x5, 0xc0014e5980, 0x11) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00427e000, 0x0, 0x0, 0xa9c5be) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00427e000, 0xc0044a6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0043e5d40, 0xc003c17378, 0x9, 0x9, 0x14, 0x7f06bcad9878, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0043e5d40, 0xc003c17378, 0x9, 0x9, 0x9, 0xa7fa45, 0xc00cd4305c, 0xc0006fa880) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc003c17378, 0x9, 0x9, 0x58a03a0, 0xc0043e5d40, 0x0, 0x0, 0xc00cd43050, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc003c17340, 0xc00cd43050, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc003b50300, 0xc00449c570, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0022b7c00, 0x5981b58, 0xc003b50300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0022b7c00, 0x5981b58, 0xc003b50300, 0x435fb83, 0x8) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 4831 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1(0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1892 +0x2b5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1700) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3734 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func1(0x59223f8, 0xc00049eb70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:754 +0xef | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc00049eb70, 0x0, 0x0, 0xc0013147c8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3588 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc00066fea0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc00066fe98) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc00066fe90, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc00066fe90, 0x5922350, 0xc00203a300, 0xc00065e3c0, 0xc00203a300) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc000ebc000, 0x5922350, 0xc00066f9b0, 0xc0005c4510, 0x1, 0x8, 0xc0005c4518, 0x1, 0x1, 0x169bb531ad9aa2aa, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:83 +0x3ca | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc000e6ccf0, 0x5922350, 0xc002f57c00, 0xc0005c4510, 0x1, 0x8, 0xc0005c4518, 0x1, 0x1, 0x169bb531ad9aa2aa, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:69 +0xb4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1(0x5922350, 0xc002f57c00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:220 +0xe3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc002f57c00, 0x0, 0x0, 0xc00066f560) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5449 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2261 +0x53 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc00118dbc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4794 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x29) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf14d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5453 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2162 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc00118dd28) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3962 [select]: | |
| github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:166 +0xea | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc002836250) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3927 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.respStreamClientAdapter.recvInternal(0x5922350, 0xc000a76a80, 0xc00065e5a0, 0xc00065e660, 0xc001732990, 0x2, 0x2, 0xc0017327e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:533 +0xb4 | |
| github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:562 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc001e7e000, 0x5922350, 0xc000a769c0, 0xc000f17a90, 0x1, 0x8, 0xc000f17a98, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:270 +0x911 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc001e7e000, 0x5922350, 0xc000a769c0, 0xc0009d0700, 0x0, 0xc0008f5020, 0xc00002b0e0, 0x1, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:151 +0x34c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1(0x5922350, 0xc000a769c0, 0xc000e3e380, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:70 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc00093f710, 0xc00093f9e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 3629 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d89720) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3632 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d897c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7300 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000ce57a0, 0x5978f10, 0xc0056bd410, 0x58e4870, 0xc000ce57a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:381 +0x210 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler(0x4281ba0, 0xc000ce57a0, 0x5965e90, 0xc0070ae7c0, 0x79a46f0, 0xc00754f888) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:168 +0xad | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x4281ba0, 0xc000ce57a0, 0x5966670, 0xc0016f6180, 0xc007779230, 0x49a99e8, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:185 +0x45b | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x4281ba0, 0xc000ce57a0, 0x5966670, 0xc0016f6180, 0x40c4c00, 0xc001479d80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x4281ba0, 0xc000ce57a0, 0x5966670, 0xc0016f6180, 0xc007779230, 0xc001479d80, 0xc001479d80, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:186 +0x99 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x4281ba0, 0xc000ce57a0, 0x5966670, 0xc0016f6180, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x1, 0x4281ba0, 0xc000ce57a0, 0x5966670, 0xc0016f6180, 0xc007779230, 0xc001479cc0, 0x40c4c00, 0xc001479cc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb7 | |
| google.golang.org/grpc.chainStreamServerInterceptors.func1(0x4281ba0, 0xc000ce57a0, 0x5966670, 0xc0016f6180, 0xc007779230, 0x49a99e8, 0x59223f8, 0xc005503890) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1401 +0xbd | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc00201ddc0, 0x5981b58, 0xc005216300, 0xc006b607e0, 0xc0038bc810, 0x765a620, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1549 +0x6d8 | |
| google.golang.org/grpc.(*Server).handleStream(0xc00201ddc0, 0x5981b58, 0xc005216300, 0xc006b607e0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1622 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc003753f70, 0xc00201ddc0, 0x5981b58, 0xc005216300, 0xc006b607e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:941 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:939 +0x1fd | |
| goroutine 3416 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026401b0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026401b0, 0xc0003a22a0, 0x1, 0x1, 0xc00165cfb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000b5a400) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 5445 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca2b350, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc003681b18, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc003681b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc003681b00, 0xc001aa0488, 0xc0054d6668, 0x203000) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc0024077e8, 0xc001aa0488, 0x42f37d, 0xc0054d6668) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc0024077e8, 0x0, 0x0, 0x7f0692c441c8, 0x28) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc005abcef8, 0x59223f8, 0xc001ca6ea0, 0xc00256de80, 0x5905230, 0xc0024077e8, 0xc003adcab0, 0xc00550af01, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:135 +0x50 | |
| github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2330 +0xfc | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc0009d2280) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3880 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0014fd230) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd230, 0x0, 0x0, 0xc003300140) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3523 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc0012ca880) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc0012ca878) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0012ca870, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc0012ca870, 0x5922350, 0xc00203a000, 0xc000110b40, 0xc00203a000) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc001e7f200, 0x5922350, 0xc0012ca750, 0xc0013a6830, 0x1, 0x8, 0xc0013a6838, 0x1, 0x1, 0x169bb531aa633dd4, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:83 +0x3ca | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc000e6de00, 0x5922350, 0xc0026d4cc0, 0xc0013a6830, 0x1, 0x8, 0xc0013a6838, 0x1, 0x1, 0x169bb531aa633dd4, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:69 +0xb4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1(0x5922350, 0xc0026d4cc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:220 +0xe3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc0026d4cc0, 0x0, 0x0, 0xc0012ca660) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4787 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x1d) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1460) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4820 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc003cf50d0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc003cf50c8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:212 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1670) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4689 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000d500f0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0048852c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc003a35860) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 4772 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xd) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1370) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3711 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0012cb470) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0012cb470, 0xc0003a22a0, 0x1, 0x1, 0xc000ea4fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d0e00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4873 [select]: | |
| github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:166 +0xea | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc004eb9710) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4776 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x14) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf13b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3670 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044aef0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5032 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func5(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1963 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc0024078c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5503 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0xd) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b310c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5229 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc0036ac900, 0xc001376820, 0xc00369a1c0, 0xf, 0xc001a495c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc0026d5240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc0026d5240, 0x0, 0x0, 0xc0013769b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3615 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002206480) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002206480, 0xc0003a22a0, 0x1, 0x1, 0xc001f317b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124b700) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4788 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x1e) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1470) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4761 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x20000003f) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf12c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3631 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d89770) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4286 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc002290240, 0xc0024b32c0, 0xc0043fef80, 0xf, 0xc000dc5500, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc001f02580) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc001f02580, 0x0, 0x0, 0xc0024b3400) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5679 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00208a370, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006bc87e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc005216d80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5887 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002e8b590, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc004884660, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc006d1f860) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 3541 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc001c77180, 0xc0026431a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3687 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002c94420) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3646 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00049e390) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00049e390, 0xc0003a22a0, 0x1, 0x1, 0xc001f19fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001c99b00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3728 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa060) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5219 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc002270240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270240, 0x0, 0x0, 0xc0024a89e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5431 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc003884f00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 3542 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc001c77220, 0xc002643200) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3637 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0036feac8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5457 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000ef0fa0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 3514 [select]: | |
| net.(*pipe).read(0xc001936100, 0xc00341c000, 0x8b5, 0x8b5, 0xc00341c005, 0x12, 0x8b0) | |
| /usr/local/go/src/net/pipe.go:159 +0x185 | |
| net.(*pipe).Read(0xc001936100, 0xc00341c000, 0x8b5, 0x8b5, 0x428e5b, 0xc0291bc7c8, 0x18) | |
| /usr/local/go/src/net/pipe.go:142 +0x56 | |
| crypto/tls.(*atLeastReader).Read(0xc0291bc7c8, 0xc00341c000, 0x8b5, 0x8b5, 0x8b0, 0xc000382000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00267c5f8, 0x58a0580, 0xc0291bc7c8, 0x426be5, 0x3da8720, 0x4119f40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00267c380, 0x7f0691997fc0, 0xc001936100, 0x5, 0xc001936100, 0x11) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00267c380, 0x0, 0x0, 0xa9c5be) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00267c380, 0xc003474000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc002216e40, 0xc00342a3b8, 0x9, 0x9, 0x14, 0xc001691e00, 0xc0000b1cf8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc002216e40, 0xc00342a3b8, 0x9, 0x9, 0x9, 0x7f06bcad8a68, 0x18, 0xc0291bc7b0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00342a3b8, 0x9, 0x9, 0x58a03a0, 0xc002216e40, 0x0, 0x0, 0xc0004220c0, 0xc0000b1d5f) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00342a380, 0xc00bfa49c0, 0xc00bfa49c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc003464000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 3417 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002640030) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002640030, 0xc0003a22a0, 0x1, 0x1, 0xc00165dfb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000b5a500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 5477 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca77060, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000b48a98, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc000b48a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc000b48a80, 0x37e11d600, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc00118dce0, 0xc004665e00, 0xc004665e08, 0x28) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc00118dce0, 0x49a9320, 0xc0009d2fc0, 0x597dab0, 0xc005abd900) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cmux.(*cMux).Serve(0xc0009d2fc0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x96 | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.2.1() | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2237 +0x2f | |
| sync.(*Once).doSlow(0xc00375aaf0, 0xc004850708) | |
| /usr/local/go/src/sync/once.go:68 +0xec | |
| sync.(*Once).Do(...) | |
| /usr/local/go/src/sync/once.go:59 | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.2(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2236 +0x74 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc003757c60) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3609 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc001c777c0, 0xc002642600) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 4810 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x33) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf15d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3697 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e8270) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e8270, 0xc0003a22a0, 0x1, 0x1, 0xc000c55fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d0100) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 5064 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc00250ca80, 0xc002eee900, 0x59223f8, 0xc003698840) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 3608 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc001c77720, 0xc0026424e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 5080 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca77c28, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00361ee98, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00361ee80, 0xc003727000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00361ee80, 0xc003727000, 0x773, 0x773, 0xc003727005, 0xc0013292d8, 0xc002e0a828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00065c698, 0xc003727000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0013bfaa0, 0xc003727000, 0x773, 0x773, 0x428e5b, 0xc017334900, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc017334900, 0xc003727000, 0x773, 0x773, 0x76e, 0xc0000a0800, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc004df9778, 0x58a0580, 0xc017334900, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc004df9500, 0x7f06919ac748, 0xc0013bfaa0, 0x5, 0xc0013bfaa0, 0xb1) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc004df9500, 0x0, 0x0, 0x4) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc004df9500, 0xc0036e6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0016fafc0, 0xc00509dc38, 0x9, 0x9, 0xc000d320c0, 0xc002e0ace0, 0x4221c5) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0016fafc0, 0xc00509dc38, 0x9, 0x9, 0x9, 0xa8, 0xc004ea82c0, 0xc002e0ad50) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00509dc38, 0x9, 0x9, 0x58a03a0, 0xc0016fafc0, 0x0, 0x0, 0xa8, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00509dc00, 0xc0020af0b0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00237b500, 0xc0020af200, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0029bc1c0, 0x5981b58, 0xc00237b500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0029bc1c0, 0x5981b58, 0xc00237b500, 0xc002049ed0, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 3622 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d89680) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4899 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0009f9e50, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0044d79e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc003b50900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5227 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0013768c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 3700 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00020aae0, 0x7f0600000435) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00020aad0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193 | |
| github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc00020aa00, 0x59223f8, 0xc00066e6f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b | |
| runtime/pprof.Do(0x5922388, 0xc00066e6f0, 0xc0003a2180, 0x1, 0x1, 0xc000c577b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc00020aa00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89 | |
| created by github.com/cockroachdb/pebble/record.NewLogWriter | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea | |
| goroutine 3718 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc00020f4e0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00020f4d0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193 | |
| github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc00020f400, 0x59223f8, 0xc0026e8a20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b | |
| runtime/pprof.Do(0x5922388, 0xc0026e8a20, 0xc0003a2180, 0x1, 0x1, 0xc00285c7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc00020f400) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89 | |
| created by github.com/cockroachdb/pebble/record.NewLogWriter | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea | |
| goroutine 3425 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc000e3e380, 0x5922388, 0xc000074108, 0xc0032985a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:235 +0x127 | |
| created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler | |
| /go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:232 +0x65 | |
| goroutine 5462 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000e46550, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc005e994a0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc004dcde00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5031 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc0022e7340) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5220 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0022702a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc0022702a0, 0x0, 0x0, 0xc0024a8a00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3964 [select, 39 minutes]: | |
| database/sql.(*DB).connectionOpener(0xc00234ef70, 0x5922350, 0xc001f03080) | |
| /usr/local/go/src/database/sql/sql.go:1133 +0xb5 | |
| created by database/sql.OpenDB | |
| /usr/local/go/src/database/sql/sql.go:740 +0x12a | |
| goroutine 5217 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowScheduler).Start.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow_scheduler.go:263 +0x13d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00279fa30) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3560 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eab10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3696 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e8300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e8300, 0xc0003a22a0, 0x1, 0x1, 0xc000c557b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d0000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4764 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x5) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf12f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3580 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00132a2a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00132a2a0, 0xc0003a22a0, 0x1, 0x1, 0xc0020e37b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000b5a300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4793 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x20) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf14c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5218 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0022701e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc0022701e0, 0x0, 0x0, 0xc0024a89c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3685 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044be10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4817 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x3b) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1640) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4765 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x7) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1300) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3539 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc001c77040, 0xc0026430e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3717 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e88d0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e88d0, 0xc0003a22a0, 0x1, 0x1, 0xc000ea7fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d1500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4769 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xf) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1340) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3569 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eaba0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4725 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1922 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc000ef29f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5236 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00090a1e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 3713 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e85a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e85a0, 0xc0003a22a0, 0x1, 0x1, 0xc000ea5fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d1000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4992 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00367fa40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 5447 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1(0x49b2640, 0xc000e29310, 0xc005a6a620, 0x59223f8, 0xc0005cdad0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:247 +0xff | |
| created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:240 +0xd0 | |
| goroutine 3643 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00049e1b0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00049e1b0, 0xc0003a22a0, 0x1, 0x1, 0xc001f087b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001c99800) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4770 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xc000000011) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1350) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4094 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc001e85e00, 0x5922350, 0xc001f6fb40, 0xc000251380, 0x5978d58, 0xc00447c320, 0xc000d48d80, 0xc0036bfdb4, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:371 +0x3bf | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1(0x5922350, 0xc001f6fb40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:125 +0x44e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc001f6fb40, 0x0, 0x0, 0xc001f6fb80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3633 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d897d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3722 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa000) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5374 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002a99380) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 4775 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x12) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf13a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5253 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc0036ac900, 0xc002537950, 0xc00369a620, 0xf, 0xc002b01a40, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc0026d5240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc0026d5240, 0x0, 0x0, 0xc002537b30) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5033 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| google.golang.org/grpc.(*Server).Serve(0xc00201ddc0, 0x5905590, 0xc002407818, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:786 +0x27f | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2232 +0x3f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00254c200) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5455 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1922 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc00118dd70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5086 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1(0x59223f8, 0xc0032c4b00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:151 +0x2ca | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc001306ee8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4821 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop(0xc0018ef800, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:626 +0x232 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1680) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4795 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x26) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf14e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5454 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func3(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2206 +0x4d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc003756920) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4958 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc003b4e000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 4792 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x22) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf14b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5444 [select]: | |
| github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:166 +0xea | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc003d0d8b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5680 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc005216d80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5375 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca77230, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00249c698, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00249c680, 0xc002f06800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00249c680, 0xc002f06800, 0x773, 0x773, 0xc002f06805, 0xc0012d6658, 0xc004666828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc002c9a830, 0xc002f06800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0023c86c0, 0xc002f06800, 0x773, 0x773, 0x428e5b, 0xc00c4caae0, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc00c4caae0, 0xc002f06800, 0x773, 0x773, 0x76e, 0xc000aaa000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0032b1078, 0x58a0580, 0xc00c4caae0, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0032b0e00, 0x7f06919ac748, 0xc0023c86c0, 0x5, 0xc0023c86c0, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0032b0e00, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0032b0e00, 0xc005be4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc002def080, 0xc003c17d18, 0x9, 0x9, 0xc004666d28, 0x420ee5, 0xc002ddc720) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc002def080, 0xc003c17d18, 0x9, 0x9, 0x9, 0xc00c4caac8, 0x481e60, 0xc002ddc720) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc003c17d18, 0x9, 0x9, 0x58a03a0, 0xc002def080, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc003c17ce0, 0xc0275006f0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002a99380, 0xc003c2bb00, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc00201ddc0, 0x5981b58, 0xc002a99380) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00201ddc0, 0x5981b58, 0xc002a99380, 0xc0033bd010, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5492 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x3) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31010) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3715 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e86f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e86f0, 0xc0003a22a0, 0x1, 0x1, 0xc000ea6fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d1300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4851 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002a99200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 3552 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc00092cfe0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00092cfd0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193 | |
| github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc00092cf00, 0x59223f8, 0xc0012ca0c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b | |
| runtime/pprof.Do(0x5922388, 0xc0012ca0c0, 0xc0003a2180, 0x1, 0x1, 0xc000a98fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc00092cf00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89 | |
| created by github.com/cockroachdb/pebble/record.NewLogWriter | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea | |
| goroutine 3572 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc003761068) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4796 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x28) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf14f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5311 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc002b050e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 4832 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1(0x59223f8, 0xc003e25ce0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_rebalancer.go:190 +0x16d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25ce0, 0x0, 0x0, 0xc0006db188) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3551 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026406c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026406c0, 0xc0003a22a0, 0x1, 0x1, 0xc000a9dfb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124ae00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 5660 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc0013b1340) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 4767 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xb) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4959 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca2b6f0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001d94118, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001d94100, 0xc0043eb200, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001d94100, 0xc0043eb200, 0x8b5, 0x8b5, 0x8b0, 0xc0043eb200, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc004033900, 0xc0043eb200, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc00ddf7d28, 0xc0043eb200, 0x8b5, 0x8b5, 0x5167a5, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc004df8978, 0x58a0580, 0xc00ddf7d28, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc004df8700, 0x58a88a0, 0xc004033900, 0x5, 0xc004033900, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc004df8700, 0x0, 0x0, 0xc0298182d0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc004df8700, 0xc00346c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc00065f560, 0xc003c16038, 0x9, 0x9, 0x7f068b6b18b8, 0x48, 0xc0291d9bb8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc00065f560, 0xc003c16038, 0x9, 0x9, 0x9, 0xc029190001, 0xc004dddd38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc003c16038, 0x9, 0x9, 0x58a03a0, 0xc00065f560, 0x0, 0xc000000000, 0xaba293, 0xc003b4e000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc003c16000, 0xc0291e7d40, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc003b4e000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 3672 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044afb0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5405 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001decaf0, 0xc002dbd888, 0x5, 0x5, 0xffc03c, 0xc01bbcbeb0, 0xc01282fc0c) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001decaf0, 0xc002dbd888, 0x5, 0x5, 0xc0075312c0, 0xc001ffad68, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc005c8a810, 0xc002dbd888, 0x5, 0x5, 0x0, 0xc007531320, 0xacfcdc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc005c8a810, 0xc002dbd888, 0x5, 0x5, 0x5, 0xc007531390, 0x426f05, 0x3da3b20) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc004c55d40, 0xc002dbd888, 0x5, 0x5, 0xc01282fc00, 0x16, 0x200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc002dbd878, 0x7fffffff, 0x19e2de2, 0x7a02de8, 0xc01282fc00, 0x8, 0x200, 0x4252220) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc002dbd878, 0xc004c55d40, 0x0, 0x0, 0x7fffffff, 0xc007531640, 0x59058c0, 0x7a02de8, 0xc02315afc0, 0x5a3df077e8a, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc002dbd878, 0x7f06bca6db20, 0xc00044af90, 0xc004c55d40, 0x0, 0x0, 0x4252220, 0xc00c4ca750, 0x7fffffff, 0xc007531640, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc001352d80, 0x4252220, 0xc00c4ca750, 0xc007531640, 0x45663c, 0x770000c0075316a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:957 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001352d80, 0xc0075316e8, 0x421105) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:808 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc004c55b00, 0xc0075317c0, 0xc007531790, 0x18, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:662 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc004c55b00, 0x4252220, 0xc00c4ca750, 0x2, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:807 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc00011b5f0, 0xc0075318c8, 0x0, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8375 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc001e7f200, 0x5922350, 0xc00203a000, 0xc0013a6830, 0x1, 0x8, 0xc0013a6838, 0x1, 0x1, 0x169bb531aa633dd4, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:270 +0x911 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc001e7f200, 0x5922350, 0xc00203a000, 0xc002942230, 0xc0023e2b00, 0xc000110ae0, 0xc00059e600, 0xaaad59, 0xc002557aa0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:151 +0x34c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1(0x5922350, 0xc00203a000, 0x0, 0x10000c00249c980) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:70 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0xc00202ffc8, 0xacfb5b) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0012ca870, 0xc005c8a630) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4096 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc003e18480, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc003cf1220) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4766 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xc000000009) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1310) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4799 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x2b) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1520) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5223 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).waitAndProcess(0xc00239a820, 0x59223f8, 0xc0022704b0, 0x212e4c0b, 0xed8ac0461, 0x0, 0xc0053a8000, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:219 +0x19d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1.1(0xc0053a8000, 0xc0033b5f80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:281 +0x78 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc0033b5f80, 0xc00261ac80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:369 +0x151 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1(0x59223f8, 0xc0022704b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:279 +0x19b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc0022704b0, 0x0, 0x0, 0xc000676200) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4800 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x27) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1530) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3915 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func8(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:849 +0x1d9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc001314a80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4780 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x16) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf13f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3501 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc0026401c0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc0026401b8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0026401b0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc0026401b0, 0x5922350, 0xc001f4a040, 0xc003b840c0, 0xc001f4a040) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc001e7e000, 0x5922350, 0xc002640120, 0xc0035200e0, 0x1, 0x8, 0xc0035200e8, 0x1, 0x1, 0x169bb531a89e469d, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:83 +0x3ca | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc000640810, 0x5922350, 0xc0014ffd40, 0xc0035200e0, 0x1, 0x8, 0xc0035200e8, 0x1, 0x1, 0x169bb531a89e469d, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:69 +0xb4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1(0x5922350, 0xc0014ffd40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:220 +0xe3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc0014ffd40, 0x0, 0x0, 0xc0026400c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3703 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc002513080, 0x5922388, 0xc000074108, 0xc001db60f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:235 +0x127 | |
| created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler | |
| /go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:232 +0x65 | |
| goroutine 3916 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func9(0x5922388, 0xc002544ec0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:872 +0x215 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc003416990) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4813 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x37) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1600) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5502 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0xc) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b310b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3911 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc0022a4340, 0x5922350, 0xc002544400, 0xc0034162d0, 0xc00002b0e0, 0xc00002b1a0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xe7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc0022a4340, 0x5922350, 0xc002544400, 0xc0034162d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:226 +0x5f8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*Factory).RangeFeed.func1(0x5922350, 0xc002544400) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:137 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc002544400, 0x0, 0x0, 0xc001314a50) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3716 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e8870) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e8870, 0xc0003a22a0, 0x1, 0x1, 0xc000ea77b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d1400) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3521 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:78 +0xc5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0030aa6a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3558 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eaaf0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4907 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).run(0xc0026322a0, 0x59223f8, 0xc0012ca8d0, 0xc000d48d80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:121 +0x296 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).Start.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:100 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc002fb6288) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4785 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x1c) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1440) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4781 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x17) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1400) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3743 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowScheduler).Start.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow_scheduler.go:263 +0x13d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0006773c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5475 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func5(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1963 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc00118de00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3709 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc000167400, 0xc000024120) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 4753 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc003e25980) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25980, 0x0, 0x0, 0xc002585f20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4063 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x59223f8, 0xc003e24de0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1339 +0x325 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc00049f380, 0x0, 0x0, 0xc0029d61a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3720 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc002f77fe0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4754 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc003e186c0, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc003cf1240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5500 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0xb) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31090) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4912 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier.(*Notifier).run(0xc003894a80, 0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier/notifier.go:136 +0xea | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc004db8240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3965 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1(0x49b2640, 0xc00122ea50, 0xc000238930, 0x59223f8, 0xc0012ca8d0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:247 +0xff | |
| created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:240 +0xd0 | |
| goroutine 5420 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca77318, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00249c618, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00249c600, 0xc004a2f800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00249c600, 0xc004a2f800, 0x773, 0x773, 0xc004a2f805, 0xc003c59918, 0xc004dde828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc002c9a828, 0xc004a2f800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc002321da0, 0xc004a2f800, 0x773, 0x773, 0x428e5b, 0xc00ddf7de8, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc00ddf7de8, 0xc004a2f800, 0x773, 0x773, 0x76e, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0024f4278, 0x58a0580, 0xc00ddf7de8, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0024f4000, 0x7f06919ac748, 0xc002321da0, 0x5, 0xc002321da0, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0024f4000, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0024f4000, 0xc004954000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc004950420, 0xc000dac818, 0x9, 0x9, 0xc004dded28, 0x420ee5, 0xc003837f20) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc004950420, 0xc000dac818, 0x9, 0x9, 0x9, 0xc00ddf7dd0, 0x481e60, 0xc003837f20) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000dac818, 0x9, 0x9, 0x58a03a0, 0xc004950420, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000dac7e0, 0xc0291e7d60, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc005216300, 0xc005296990, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc00201ddc0, 0x5981b58, 0xc005216300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00201ddc0, 0x5981b58, 0xc005216300, 0xc002800570, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5108 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc0036ac900, 0xc001b6b950, 0xc0013a7130, 0xf, 0xc000022300, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc0026d5240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc0026d5240, 0x0, 0x0, 0xc001b6ba90) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3654 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.newInfoStore.func1(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/infostore.go:190 +0x14d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc00387f220) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5470 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc005c64b40, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc003b30fa0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3555 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eaac0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3695 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e8060) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e8060, 0xc0003a22a0, 0x1, 0x1, 0xc000c54fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000575000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3667 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0012cb080) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0012cb080, 0xc0003a22a0, 0x1, 0x1, 0xc0009c0fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001474d00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4728 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc002247340) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 3689 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002c94450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4919 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0020fdc40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3549 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002640570) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002640570, 0xc0003a22a0, 0x1, 0x1, 0xc0012887b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124ab00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4782 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x19) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1410) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3692 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e8120) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e8120, 0xc0003a22a0, 0x1, 0x1, 0xc000fa2fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000574d00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 5456 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*loopbackListener).Accept(0xc005ac36e0, 0x49b0428, 0xc001f46fc0, 0x58f6560, 0xc005ac36e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/loopback.go:49 +0xb5 | |
| google.golang.org/grpc.(*Server).Serve(0xc001f46fc0, 0x58f6560, 0xc005ac36e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:786 +0x27f | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func2(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1930 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc00118dda0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4789 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x1a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1480) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4779 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x18) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf13e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4809 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xc000000032) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf15c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5216 [select, 9 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*TemporaryObjectCleaner).Start.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/temporary_schema.go:597 +0x2de | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc000641068) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5112 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002574550, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0050fde60, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc004e04d20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 5325 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc0013b0dc0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 4090 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:113 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc0026c5540) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3967 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2261 +0x53 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc000ef2858) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4758 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xc000000000) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1290) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4186 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca2bd48, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000c08398, 0x72, 0x4a00, 0x4acb, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000c08380, 0xc0023d4000, 0x4acb, 0x4acb, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000c08380, 0xc0023d4000, 0x4acb, 0x4acb, 0x4ac6, 0xc0023d4000, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000116b80, 0xc0023d4000, 0x4acb, 0x4acb, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc00c4caa38, 0xc0023d4000, 0x4acb, 0x4acb, 0x5167a5, 0xc000aaa000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00485a278, 0x58a0580, 0xc00c4caa38, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00485a000, 0x58a88a0, 0xc000116b80, 0x5, 0xc000116b80, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00485a000, 0x0, 0x0, 0xc0275065d0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00485a000, 0xc004a94000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0046ff860, 0xc0022a6b98, 0x9, 0x9, 0x7f068c553b20, 0x48, 0xc027849a38) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0046ff860, 0xc0022a6b98, 0x9, 0x9, 0x9, 0xc023150001, 0xc003dccd38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0022a6b98, 0x9, 0x9, 0x58a03a0, 0xc0046ff860, 0x0, 0xc000000000, 0xaba293, 0xc001ace780) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0022a6b60, 0xc0275006c0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001ace780) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 4688 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca2c0e8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00107ae98, 0x72, 0x4600, 0x4644, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00107ae80, 0xc0048da000, 0x4644, 0x4644, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00107ae80, 0xc0048da000, 0x4644, 0x4644, 0x463f, 0xc0048da000, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc0005ce988, 0xc0048da000, 0x4644, 0x4644, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc006c4dd58, 0xc0048da000, 0x4644, 0x4644, 0x463f, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc003d86978, 0x58a0580, 0xc006c4dd58, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc003d86700, 0x58a88a0, 0xc0005ce988, 0x5, 0xc0005ce988, 0x9b) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc003d86700, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc003d86700, 0xc004894000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0048851a0, 0xc0038c0ab8, 0x9, 0x9, 0x4221c5, 0xc00447e4e0, 0xc003318ce8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0048851a0, 0xc0038c0ab8, 0x9, 0x9, 0x9, 0xc001c4d380, 0x5318d9, 0xc004472788) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0038c0ab8, 0x9, 0x9, 0x58a03a0, 0xc0048851a0, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0038c0a80, 0xc003c791a0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc003a35860) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 4768 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x3) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1330) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4786 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x1f) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5392 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000dfb720) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 5482 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:78 +0xc5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc00487c440) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3928 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc00013a000, 0x6, 0xc003284780, 0x5970988, 0xc0034007a0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:257 +0x670 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:143 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc00250e800, 0xc003284780, 0x5970988, 0xc0034007a0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2530 +0x136 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc001e268c0, 0xc003284780, 0x5970988, 0xc0034007a0, 0xc001189768) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:216 +0xce | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc00283cc00, 0xc003284780, 0x5970988, 0xc0034007a0, 0xc001189768, 0x199c23a) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:1101 +0x54 | |
| github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func1(0xc0011fe390, 0x596bed8, 0xc00283cc00, 0xc003284780, 0x5922350, 0xc000a76a80, 0xc00065e5a0, 0xc00065e660) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:590 +0xc5 | |
| created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:588 +0x12e | |
| goroutine 3562 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eab30) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4726 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*loopbackListener).Accept(0xc002fb2cf0, 0x49b0428, 0xc0022b7c00, 0x58f6560, 0xc002fb2cf0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/loopback.go:49 +0xb5 | |
| google.golang.org/grpc.(*Server).Serve(0xc0022b7c00, 0x58f6560, 0xc002fb2cf0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:786 +0x27f | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func2(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1930 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc000ef2a08) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3565 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eab60) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3547 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00132a090) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00132a090, 0xc0003a22a0, 0x1, 0x1, 0xc001289fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124a900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 5034 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca2b438, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc003681a98, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc003681a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc003681a80, 0x37e11d600, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc0024077d0, 0xc004de5e00, 0xc004de5e08, 0x28) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc0024077d0, 0x49a9320, 0xc00261b600, 0x597dab0, 0xc0003a47d0) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cmux.(*cMux).Serve(0xc00261b600, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x96 | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.2.1() | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2237 +0x2f | |
| sync.(*Once).doSlow(0xc000a17300, 0xc00493cf08) | |
| /usr/local/go/src/sync/once.go:68 +0xec | |
| sync.(*Once).Do(...) | |
| /usr/local/go/src/sync/once.go:59 | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.2(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2236 +0x74 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00254c220) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3684 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044be00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5661 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc002290000, 0xc000f49400, 0xc00610c690, 0xf, 0xc001ea3c80, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc0039efe00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc0039efe00, 0x0, 0x0, 0xc000f49540) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3751 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa0d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3556 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eaad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3931 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc0022a60e0, 0x5922350, 0xc000a76c00, 0xc000a76c00, 0xc0011fe440) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc0022a60e0, 0x5922350, 0xc000a76c00, 0x6) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:276 +0xd4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x5922350, 0xc000a76ac0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:263 +0x6d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc000a76ac0, 0x0, 0x0, 0xc003760ac8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5493 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0xc000000004) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31020) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5072 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0022700c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc0022700c0, 0x0, 0x0, 0xc0024a8980) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3710 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e87e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e87e0, 0xc0003a22a0, 0x1, 0x1, 0xc000ea47b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d0d00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3875 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0014fcfc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fcfc0, 0x0, 0x0, 0xc0033000a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4819 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x3e) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1660) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3707 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc000167220, 0xc000499f20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3414 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026400c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026400c0, 0xc0003a22a0, 0x1, 0x1, 0xc0009c37b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000b5a100) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4808 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x31) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf15b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5210 [select, 4 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).run(0xc000ffd500, 0x59223f8, 0xc001ca6ea0, 0xc00256de80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:121 +0x296 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).Start.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:100 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc000640768) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4763 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x200000004) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf12e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3963 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca2bc60, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0030a4598, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc0030a4580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc0030a4580, 0x37e11d600, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc0030b4288, 0xc0036b2e10, 0xc0036b2e18, 0x38) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc0030b4288, 0x49aea28, 0xc000e3e380, 0x59223f8, 0xc001ca68a0) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc000c62680, 0x59223f8, 0xc001ca68a0, 0xc000e3e380, 0x5905230, 0xc0030b4288, 0xc003e24630, 0xf, 0x437242e) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:135 +0x50 | |
| github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2330 +0xfc | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc001f02f40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3694 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e81e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e81e0, 0xc0003a22a0, 0x1, 0x1, 0xc000c547b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000574f00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3966 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:113 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc002584520) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3688 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002c94438) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3763 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| crypto/tls.(*listener).Accept(0xc0030b41e0, 0x40dbce0, 0xc0014fc270, 0x3d1ed60, 0x7646620) | |
| /usr/local/go/src/crypto/tls/tls.go:67 +0x37 | |
| net/http.(*Server).Serve(0xc00309c620, 0x58ef210, 0xc0030b41e0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2961 +0x285 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func4(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2309 +0x3f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0030b41f8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3698 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e83c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e83c0, 0xc0003a22a0, 0x1, 0x1, 0xc000c567b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d0300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3913 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).PeriodicallyRefreshSomeLeases.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1007 +0x2ec | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc000677460) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4749 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc003e257d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e257d0, 0x0, 0x0, 0xc002585ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4724 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func3(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2206 +0x4d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc002584680) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3554 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eaab0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3508 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc0012cadc0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc0012cadb8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0012cadb0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc0012cadb0, 0x5922350, 0xc001304000, 0xc0001112c0, 0xc001304000) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc000dffb00, 0x5922350, 0xc0012ca990, 0xc0034f4c00, 0x1, 0x8, 0xc0034f4c08, 0x1, 0x1, 0x169bb531ac11f60e, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:83 +0x3ca | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc0026f7470, 0x5922350, 0xc0039ef6c0, 0xc0034f4c00, 0x1, 0x8, 0xc0034f4c08, 0x1, 0x1, 0x169bb531ac11f60e, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:69 +0xb4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1(0x5922350, 0xc0039ef6c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:220 +0xe3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc0039ef6c0, 0x0, 0x0, 0xc0012ca8a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3520 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca2bb78, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000e3e298, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc000e3e280, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc000e3e280, 0x37e11d600, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc003a021b0, 0xc003103e00, 0xc003103e08, 0x28) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc003a021b0, 0x49a9320, 0xc002f57e00, 0x597dab0, 0xc002a661b0) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cmux.(*cMux).Serve(0xc002f57e00, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x96 | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.2.1() | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2237 +0x2f | |
| sync.(*Once).doSlow(0xc0005c5620, 0xc001f1af08) | |
| /usr/local/go/src/sync/once.go:68 +0xec | |
| sync.(*Once).Do(...) | |
| /usr/local/go/src/sync/once.go:59 | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.2(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2236 +0x74 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0032b6900) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3881 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).waitAndProcess(0xc000144820, 0x59223f8, 0xc0014fd350, 0xb134a34, 0xed8ac044a, 0x0, 0xc002dc8a80, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:219 +0x19d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1.1(0xc002dc8a80, 0xc0014fce01) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:281 +0x78 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc0014fce70, 0xc01de7a7c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:369 +0x151 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1(0x59223f8, 0xc0014fd350) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:279 +0x19b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd350, 0x0, 0x0, 0xc0013240b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4901 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca2b7d8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001d65b98, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001d65b80, 0xc004a2e000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001d65b80, 0xc004a2e000, 0x773, 0x773, 0xc004a2e005, 0xc000f383d8, 0xc003dc9828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000011b10, 0xc004a2e000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0048fefc0, 0xc004a2e000, 0x773, 0x773, 0x428e5b, 0xc017fa2570, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc017fa2570, 0xc004a2e000, 0x773, 0x773, 0xc007e34a28, 0xc000382400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc003c033f8, 0x58a0580, 0xc017fa2570, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc003c03180, 0x7f06919ac748, 0xc0048fefc0, 0x5, 0xc0048fefc0, 0xc003dc99c8) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc003c03180, 0x0, 0x0, 0xc01bdd1d00) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc003c03180, 0xc004a38000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0044d7980, 0xc003c17538, 0x9, 0x9, 0xc017fa2558, 0xc003dc9ce0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0044d7980, 0xc003c17538, 0x9, 0x9, 0x9, 0x1, 0xc017fa2558, 0x0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc003c17538, 0x9, 0x9, 0x58a03a0, 0xc0044d7980, 0x0, 0xc000000000, 0x100, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc003c17500, 0xc00449d8c0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc003b50900, 0xc00449da10, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0029bc1c0, 0x5981b58, 0xc003b50900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0029bc1c0, 0x5981b58, 0xc003b50900, 0xc004887510, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 4910 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage.(*Storage).deleteSessionsLoop(0xc0039a0f20, 0x59223f8, 0xc002ad7540) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage/slstorage.go:317 +0x1fa | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc004db81d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3779 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:380 +0x5a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0032b6980) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4822 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop(0xc0018ef800, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:688 +0x10e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1690) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5221 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc002270300) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270300, 0x0, 0x0, 0xc0024a8a20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3706 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc000167180, 0xc000499da0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3648 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00049e3f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00049e3f0, 0xc0003a22a0, 0x1, 0x1, 0xc001f157b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001c99d00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4828 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startLeaseRenewer.func1(0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1783 +0x2b0 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf16f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3749 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa0b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5701 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc002c3d0e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 4739 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0016922d0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 3714 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e8750) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e8750, 0xc0003a22a0, 0x1, 0x1, 0xc000ea67b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d1200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4791 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x21) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf14a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3747 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa090) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3708 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc0001672c0, 0xc000024060) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 5268 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0012aadc0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 4707 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Sender).Run.func2(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:253 +0x1a9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc000121880) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3705 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc0001670e0, 0xc000499c20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3748 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa0a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3649 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00049e450) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00049e450, 0xc0003a22a0, 0x1, 0x1, 0xc001f197b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001c99e00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4893 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*jobScheduler).runDaemon.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/job_scheduler.go:357 +0x272 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc002eb1110) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4060 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:78 +0xc5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0029d60d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3704 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.newInfoStore.func1(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/infostore.go:190 +0x14d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0031eab60) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3746 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3781 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x59223f8, 0xc0015de330) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1374 +0x30a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0015de330, 0x0, 0x0, 0xc0030aa760) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5461 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000f48410, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc005e99440, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc001b3bc20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 3502 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1(0x5922350, 0xc001f4a040, 0xc000e3e380, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:66 +0x12f | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0026401b0, 0xc002640210) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 5664 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000f49b80, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0063591a0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc005f28900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5665 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc005f28900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 434723 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0009319e0, 0x1067) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0009319d0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:193 | |
| github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc000931900, 0x59223f8, 0xc00e05cf60) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:396 +0x10b | |
| runtime/pprof.Do(0x5922388, 0xc00e05cf60, 0xc0003a2180, 0x1, 0x1, 0xc0044c2fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble/record.NewLogWriter.func2(0xc000931900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:314 +0x89 | |
| created by github.com/cockroachdb/pebble/record.NewLogWriter | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/record/log_writer.go:313 +0x2ea | |
| goroutine 4778 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x23) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf13d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7301 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0024e0500, 0xc007779228, 0x5, 0x5, 0xc00754f978, 0x12, 0x34) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:177 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0024e0500, 0xc007779228, 0x5, 0x5, 0xc017fee700, 0xc004cc1170, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:171 +0x211 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0055037a0, 0xc007779228, 0x5, 0x5, 0x1c21585, 0xc00754fa40, 0xad11dc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc0055037a0, 0xc007779228, 0x5, 0x5, 0x5, 0xc00754fab0, 0x426f05, 0x3da3b20) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc006b607e0, 0xc007779228, 0x5, 0x5, 0x41e270, 0xc00754fc68, 0xc003684ec0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc007779218, 0x7fffffff, 0xc00754fbe8, 0xc000997bb8, 0xd44dc3c432f1de01, 0xd40000c005296900, 0x1, 0xc000997ba0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc007779218, 0xc006b607e0, 0x0, 0x0, 0x7fffffff, 0xc00754fd78, 0x59058c0, 0x7a02de8, 0x45663c, 0xc00754fc48, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc007779218, 0x7f06bca6db20, 0xc00044af90, 0xc006b607e0, 0x0, 0x0, 0x41b3100, 0xc01ca2b278, 0x7fffffff, 0xc00754fd78, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc0016f6180, 0x41b3100, 0xc01ca2b278, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:1545 +0x156 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc0056bd410, 0x593db70, 0xc004e04930, 0xc017fee700) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:187 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:83 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1(0x5978f10, 0xc0056bd410, 0xc000ce57a0, 0x593db70, 0xc004e04930, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:340 +0x82 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1(0x593db70, 0xc004e04930) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:376 +0x5d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x593db70, 0xc004e04930, 0xc004e04780, 0x0, 0xc005503980) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4180 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002405360, 0xc002607120, 0x5, 0x5, 0xc012dd9392, 0xc00919a000, 0x40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:177 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002405360, 0xc002607120, 0x5, 0x5, 0x0, 0xc0005bfd28, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:171 +0x211 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00397ec30, 0xc002607120, 0x5, 0x5, 0x7a02de8, 0xc0059477c8, 0xad11dc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc00397ec30, 0xc002607120, 0x5, 0x5, 0x5, 0x3bf3df4db666cd01, 0x3b0000c003c79740, 0x0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001aa18c0, 0xc002607120, 0x5, 0x5, 0xc0065b8788, 0x7a02d01, 0x3c55740) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc002607110, 0x7fffffff, 0x3e62b80, 0xc0021c1ce0, 0x1, 0xc01766f800, 0x5a3ce40ec00, 0xc000c81400) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc002607110, 0xc001aa18c0, 0x0, 0x0, 0x7fffffff, 0xc005947b00, 0x59058c0, 0x7a02de8, 0x40, 0x40, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc002607110, 0x7f06bca6db20, 0xc00044af90, 0xc001aa18c0, 0x0, 0x0, 0x41ae740, 0xc01c12a2d0, 0x7fffffff, 0xc005947b00, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc001d70900, 0x41ae740, 0xc01c12a2d0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:1545 +0x156 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc000698fc0, 0x0, 0x1abc4e0, 0x49a94e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:679 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc003298780, 0x5922350, 0xc0009d2600, 0xc002c9a6e8, 0xc0009d2640, 0xc005947f08, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:331 +0x387 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3(0x5922350, 0xc0009d2600) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:139 +0x9e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc0009d2600, 0x0, 0x0, 0xc0009d26c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3702 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x5922350, 0xc0039efe00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:403 +0x51 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc0039efe00, 0x0, 0x0, 0xc002c94a80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3557 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eaae0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5499 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0xc00000000a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4784 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x15) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1430) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5498 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x9) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31070) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7281 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc002290240, 0xc002fad180, 0xc00011f060, 0xf, 0xc005d281e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc001f02580) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc001f02580, 0x0, 0x0, 0xc002fad2c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5714 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046ab8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001462d98, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001462d80, 0xc0018a5800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001462d80, 0xc0018a5800, 0x773, 0x773, 0xc0018a5805, 0xc0024a6dd8, 0xc006341828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc0005cfc68, 0xc0018a5800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc006359140, 0xc0018a5800, 0x773, 0x773, 0x428e5b, 0xc018493a58, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc018493a58, 0xc0018a5800, 0x773, 0x773, 0xc0072d4a28, 0xc002310400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0065b8278, 0x58a0580, 0xc018493a58, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0065b8000, 0x7f06919ac748, 0xc006359140, 0x5, 0xc006359140, 0xc0063419c8) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0065b8000, 0x0, 0x0, 0xc001b02900) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0065b8000, 0xc005f46000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc005f40720, 0xc002f66118, 0x9, 0x9, 0xc018493a40, 0xc006341ce0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc005f40720, 0xc002f66118, 0x9, 0x9, 0x9, 0x1, 0xc018493a40, 0x0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc002f66118, 0x9, 0x9, 0x58a03a0, 0xc005f40720, 0x0, 0xc000000000, 0x103, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc002f660e0, 0xc005f30ab0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc005f28900, 0xc005f30c00, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0029bc1c0, 0x5981b58, 0xc005f28900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0029bc1c0, 0x5981b58, 0xc005f28900, 0xc0063527d0, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5222 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc002270390) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270390, 0x0, 0x0, 0xc0024a8a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5708 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc002dec0e0, 0x5922350, 0xc0064e8b40, 0xc0064e8b40, 0xc006480ee0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc002dec0e0, 0x5922350, 0xc0064e8b40, 0x10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:276 +0xd4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x5922350, 0xc003665740) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:263 +0x6d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc003665740, 0x0, 0x0, 0xc005f36258) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3750 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa0c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3742 [select, 9 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*TemporaryObjectCleaner).Start.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/temporary_schema.go:597 +0x2de | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc001314990) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4702 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca2b8c0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000c08718, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000c08700, 0xc004b8c000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000c08700, 0xc004b8c000, 0x8b5, 0x8b5, 0x20300700000000, 0x7f067f3eefff, 0xc0049477e8) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000116cb0, 0xc004b8c000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc01ca09c08, 0xc004b8c000, 0x8b5, 0x8b5, 0x1, 0xc000382000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00485a978, 0x58a0580, 0xc01ca09c08, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00485a700, 0x58a88a0, 0xc000116cb0, 0x5, 0xc000116cb0, 0xc004947960) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00485a700, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00485a700, 0xc004ba2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc004b89d40, 0xc00375f618, 0x9, 0x9, 0x0, 0xc004947c98, 0x58e6b70) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc004b89d40, 0xc00375f618, 0x9, 0x9, 0x9, 0xc0000f7558, 0x9e7e783d60483a01, 0xc02ad472f0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00375f618, 0x9, 0x9, 0x58a03a0, 0xc004b89d40, 0x0, 0x0, 0xc01d312a28, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00375f5e0, 0xc02ad47290, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc003dbe1e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 3745 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc00254d3a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5074 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:380 +0x5a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0030132a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3559 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eab00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3641 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.startPurgeOldSessions.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/purge_auth_session.go:65 +0x225 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0037573a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3752 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc002fb7f20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3729 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0030aa070) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3639 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0036feaf8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3876 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0014fd080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd080, 0x0, 0x0, 0xc0033000c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4852 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca2bf18, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001d65798, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001d65780, 0xc002f07000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001d65780, 0xc002f07000, 0x773, 0x773, 0xc002f07005, 0xc0024bd058, 0xc004501828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc0000119a8, 0xc002f07000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc003c6e720, 0xc002f07000, 0x773, 0x773, 0x428e5b, 0xc0014818f0, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc0014818f0, 0xc002f07000, 0x773, 0x773, 0x76e, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc003c03078, 0x58a0580, 0xc0014818f0, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc003c02e00, 0x7f06919ac748, 0xc003c6e720, 0x5, 0xc003c6e720, 0xc6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc003c02e00, 0x0, 0x0, 0x4) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc003c02e00, 0xc0048b6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc004885920, 0xc0038c0c78, 0x9, 0x9, 0xc0046818c0, 0xc004501ce0, 0x4221c5) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc004885920, 0xc0038c0c78, 0x9, 0x9, 0x9, 0x16d, 0xc0037b1200, 0xc004501d50) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0038c0c78, 0x9, 0x9, 0x58a03a0, 0xc004885920, 0x0, 0x0, 0xbd, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0038c0c40, 0xc003c79650, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002a99200, 0xc003c797d0, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0029bc1c0, 0x5981b58, 0xc002a99200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0029bc1c0, 0x5981b58, 0xc002a99200, 0xc0038d5ce0, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5486 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x59223f8, 0xc002207fb0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1374 +0x30a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc002207fb0, 0x0, 0x0, 0xc003db4040) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3611 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0022063c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0022063c0, 0xc0003a22a0, 0x1, 0x1, 0xc001f2c7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124b300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3439 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:113 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc003300020) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3652 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x5922350, 0xc0026d5240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:403 +0x51 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc0026d5240, 0x0, 0x0, 0xc0036ff170) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3415 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002640120) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002640120, 0xc0003a22a0, 0x1, 0x1, 0xc001f0d7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000b5a200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4916 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0020fdc00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3712 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e8630) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e8630, 0xc0003a22a0, 0x1, 0x1, 0xc000ea57b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d0f00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 5039 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:113 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc00254c5a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4062 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:380 +0x5a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc001396cc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3634 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d897e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4790 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x24) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1490) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3874 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0014fcf60) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fcf60, 0x0, 0x0, 0xc003300080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4757 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xc000000002) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1280) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5458 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001dab400, 0xc0026070c0, 0x5, 0x5, 0xffc03c, 0xc00555abf0, 0xc01dff460c) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001dab400, 0xc0026070c0, 0x5, 0x5, 0xc003bfd2c0, 0xc0036fa828, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc005c6a2d0, 0xc0026070c0, 0x5, 0x5, 0x0, 0xc003bfd320, 0xacfcdc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc005c6a2d0, 0xc0026070c0, 0x5, 0x5, 0x5, 0xc003bfd390, 0x426f05, 0x3da3b20) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc005c64360, 0xc0026070c0, 0x5, 0x5, 0xc01dff4600, 0x16, 0x200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc0026070b0, 0x7fffffff, 0x19e2de2, 0x7a02de8, 0xc01dff4600, 0x8, 0x200, 0x4252220) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc0026070b0, 0xc005c64360, 0x0, 0x0, 0x7fffffff, 0xc003bfd640, 0x59058c0, 0x7a02de8, 0xc027d33260, 0x5a3df078114, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc0026070b0, 0x7f06bca6db20, 0xc00044af90, 0xc005c64360, 0x0, 0x0, 0x4252220, 0xc01dbfcb28, 0x7fffffff, 0xc003bfd640, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc00249de00, 0x4252220, 0xc01dbfcb28, 0xc003bfd640, 0x45663c, 0xc003bfd6a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:957 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00249de00, 0xc003bfd6e8, 0x421105) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:808 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc005c64120, 0xc003bfd7c0, 0xc003bfd790, 0x18, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:662 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc005c64120, 0x4252220, 0xc01dbfcb28, 0x2, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:807 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc003b303e0, 0xc003bfd8c8, 0x0, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8375 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc000dffb00, 0x5922350, 0xc001304000, 0xc0034f4c00, 0x1, 0x8, 0xc0034f4c08, 0x1, 0x1, 0x169bb531ac11f60e, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:270 +0x911 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc000dffb00, 0x5922350, 0xc001304000, 0xc002eb2380, 0x53a100, 0xc000110b40, 0xc00059f680, 0xc0000b2290, 0xc00053ffb0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:151 +0x34c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1(0x5922350, 0xc001304000, 0x0, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:70 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0xc003b50900, 0x100010000) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0012cadb0, 0xc005c6a0f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 3606 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc001c77360, 0xc002642180) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3900 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.respStreamClientAdapter.recvInternal(0x5922350, 0xc001b6d400, 0xc001cc7e60, 0xc001cc7ec0, 0xc00330d990, 0x2, 0x2, 0xc00330d7e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:533 +0xb4 | |
| github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:562 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc001e7e000, 0x5922350, 0xc001f4a040, 0xc0035200e0, 0x1, 0x8, 0xc0035200e8, 0x1, 0x1, 0x169bb531a89e469d, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:270 +0x911 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc001e7e000, 0x5922350, 0xc001f4a040, 0xc0020b3d50, 0xc0033bc000, 0xc003b84000, 0xc0002ba600, 0x0, 0xc00331c6b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:151 +0x34c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1(0x5922350, 0xc001f4a040, 0xc000e3e380, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:70 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0026401b0, 0xc0022070b0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 3636 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0036feab0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4866 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).waitAndProcess(0xc000d529c0, 0x59223f8, 0xc003e25a70, 0xff4870f, 0xed8ac0461, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:219 +0x19d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1(0x59223f8, 0xc003e25a70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:286 +0x309 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a70, 0x0, 0x0, 0xc00447c480) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3907 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).startSQLStatsFlushLoop.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/provider.go:98 +0x1c5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0013149c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4850 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000d50550, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0045e0f60, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc002a99200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5630 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046f40, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc002352a98, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc002352a80, 0xc00343d800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc002352a80, 0xc00343d800, 0x773, 0x773, 0xc00343d805, 0xc000523558, 0xc0044d3828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc002c9b060, 0xc00343d800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc004b93260, 0xc00343d800, 0x773, 0x773, 0x428e5b, 0xc01dbfcc30, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc01dbfcc30, 0xc00343d800, 0x773, 0x773, 0x76e, 0xc000aaa000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0024f45f8, 0x58a0580, 0xc01dbfcc30, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0024f4380, 0x7f06919ac748, 0xc004b93260, 0x5, 0xc004b93260, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0024f4380, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0024f4380, 0xc006538000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0065153e0, 0xc00342b0d8, 0x9, 0x9, 0xc0044d3d28, 0x420ee5, 0xc0059d53e0) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0065153e0, 0xc00342b0d8, 0x9, 0x9, 0x9, 0xc01dbfcc18, 0x481e60, 0xc0059d53e0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00342b0d8, 0x9, 0x9, 0x58a03a0, 0xc0065153e0, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00342b0a0, 0xc027d35a00, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0060d3380, 0xc00652e7e0, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0022b7c00, 0x5981b58, 0xc0060d3380) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0022b7c00, 0x5981b58, 0xc0060d3380, 0xc002801760, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5029 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*loopbackListener).Accept(0xc00353f6b0, 0x49b0428, 0xc00201ddc0, 0x58f6560, 0xc00353f6b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/loopback.go:49 +0xb5 | |
| google.golang.org/grpc.(*Server).Serve(0xc00201ddc0, 0x58f6560, 0xc00353f6b0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:786 +0x27f | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func2(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1930 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002407860) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3545 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00132a0f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00132a0f0, 0xc0003a22a0, 0x1, 0x1, 0xc000a9c7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124a700) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4803 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x2c) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1560) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3610 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002206300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002206300, 0xc0003a22a0, 0x1, 0x1, 0xc001f2a7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124b200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3879 [select, 38 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0014fd1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd1a0, 0x0, 0x0, 0xc003300120) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7256 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0068ced80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5071 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc002270060) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270060, 0x0, 0x0, 0xc0024a8960) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3561 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0031eab20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3546 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002640660) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002640660, 0xc0003a22a0, 0x1, 0x1, 0xc00128f7b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124a800) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3901 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc00015ca80, 0x10, 0xc0033a30e0, 0x5970988, 0xc003013e60, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:257 +0x670 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:143 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc00250e800, 0xc0033a30e0, 0x5970988, 0xc003013e60, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2530 +0x136 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc001e268c0, 0xc0033a30e0, 0x5970988, 0xc003013e60, 0xc00331c768) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:216 +0xce | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc00283cc00, 0xc0033a30e0, 0x5970988, 0xc003013e60, 0xc00331c768, 0x199c23a) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:1101 +0x54 | |
| github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func1(0xc002694d30, 0x596bed8, 0xc00283cc00, 0xc0033a30e0, 0x5922350, 0xc001b6d400, 0xc001cc7e60, 0xc001cc7ec0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:590 +0xc5 | |
| created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:588 +0x12e | |
| goroutine 3607 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc001c774a0, 0xc002642480) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 5659 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000f49450) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 3612 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002206420) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002206420, 0xc0003a22a0, 0x1, 0x1, 0xc001f2afb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124b400) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 5501 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x8) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b310a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3665 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00011c510) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00011c510, 0xc0003a22a0, 0x1, 0x1, 0xc0009c5fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001474b00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3877 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0014fd0e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd0e0, 0x0, 0x0, 0xc0033000e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5246 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00090a3c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 4709 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1(0x59223f8, 0xc003c7c0c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:151 +0x2ca | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc00136ec78) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3635 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d897f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3699 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026e8420) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026e8420, 0xc0003a22a0, 0x1, 0x1, 0xc000c56fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0012d0400) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4771 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xc000000013) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1360) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3671 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00044af00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3613 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002206360) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002206360, 0xc0003a22a0, 0x1, 0x1, 0xc001f2efb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124b500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3570 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc003761038) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3544 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc002640510) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc002640510, 0xc0003a22a0, 0x1, 0x1, 0xc000a977b8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00124a600) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3759 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:113 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc002fe5c20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4064 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x59223f8, 0xc00049f380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1374 +0x30a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc00049f380, 0x0, 0x0, 0xc0029d61b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4914 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowScheduler).Start.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow_scheduler.go:263 +0x13d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc004db8280) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4755 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func1(0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:193 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc0006db128) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5326 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc002290000, 0xc000f48500, 0xc0034f5b20, 0xf, 0xc001ea2ae0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc0039efe00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc0039efe00, 0x0, 0x0, 0xc000f48640) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4762 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x6e65000200000008) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf12d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5076 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x59223f8, 0xc0033b5710) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1374 +0x30a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc0033b5710, 0x0, 0x0, 0xc0053b9f00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5490 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x100000001) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b30ff0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5110 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc004e04d20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 4759 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x6) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf12a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3735 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache.(*Cache).periodicallyRefreshProtectedtsCache(0xc0033a8bb0, 0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache/cache.go:160 +0x239 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0006762d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3538 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.newInfoStore.func1(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/infostore.go:190 +0x14d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc003685d40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3626 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d896f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5107 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc000e70000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 3571 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc003761050) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5079 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00237b500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 4818 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x3c) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1650) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3753 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:134 +0xc7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc002fb7f38) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3762 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| net/http.(*Server).Serve(0xc003302000, 0x5905590, 0xc0030b4198, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2961 +0x285 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func3(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2296 +0x127 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0008b4570) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4729 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func5(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1963 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc000ef2a68) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5067 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00124d180, 0xc00263b378, 0x5, 0x5, 0xc01c9f6c12, 0xc0231ee000, 0x40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:177 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00124d180, 0xc00263b378, 0x5, 0x5, 0x0, 0xc00374c078, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:171 +0x211 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc003698bd0, 0xc00263b378, 0x5, 0x5, 0x7a02de8, 0xc008bcb7c8, 0xad11dc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc003698bd0, 0xc00263b378, 0x5, 0x5, 0x5, 0xafaa2c15eab17101, 0xaf0000c0020af1a0, 0x3) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc002eeec60, 0xc00263b378, 0x5, 0x5, 0xc0065b87a0, 0x7a02d01, 0x3c55740) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc00263b368, 0x7fffffff, 0x3e62b80, 0xc0052c1338, 0x1, 0xc01cae2c00, 0x5a3dae6bce4, 0xc0000a0800) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc00263b368, 0xc002eeec60, 0x0, 0x0, 0x7fffffff, 0xc008bcbb00, 0x59058c0, 0x7a02de8, 0x40, 0x40, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc00263b368, 0x7f06bca6db20, 0xc00044af90, 0xc002eeec60, 0x0, 0x0, 0x41ae740, 0xc01e60d9f0, 0x7fffffff, 0xc008bcbb00, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc003c01c80, 0x41ae740, 0xc01e60d9f0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:1545 +0x156 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc00089a9b0, 0x0, 0x1a487e0, 0x49a94e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:679 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc003298780, 0x5922350, 0xc001f4b080, 0xc004033b10, 0xc001f4b100, 0xc008bcbf08, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:331 +0x387 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3(0x5922350, 0xc001f4b080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:139 +0x9e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc001f4b080, 0x0, 0x0, 0xc001f4b1c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4706 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*jobScheduler).runDaemon.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/job_scheduler.go:357 +0x272 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc00136ec30) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3624 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:108 +0x11e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc001d896d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4059 [select]: | |
| net.(*pipe).read(0xc0014e5900, 0xc0043ea000, 0x8b5, 0x8b5, 0xc0043ea005, 0x12, 0x8b0) | |
| /usr/local/go/src/net/pipe.go:159 +0x185 | |
| net.(*pipe).Read(0xc0014e5900, 0xc0043ea000, 0x8b5, 0x8b5, 0x428e5b, 0xc01727b7b8, 0x18) | |
| /usr/local/go/src/net/pipe.go:142 +0x56 | |
| crypto/tls.(*atLeastReader).Read(0xc01727b7b8, 0xc0043ea000, 0x8b5, 0x8b5, 0x8b0, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00427e5f8, 0x58a0580, 0xc01727b7b8, 0x426be5, 0x3da8720, 0x4119f40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00427e380, 0x7f0691997fc0, 0xc0014e5900, 0x5, 0xc0014e5900, 0x11) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00427e380, 0x0, 0x0, 0xa9c5be) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00427e380, 0xc004484000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0016f35c0, 0xc0023df7d8, 0x9, 0x9, 0x14, 0xc001d68000, 0xc004504cf8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0016f35c0, 0xc0023df7d8, 0x9, 0x9, 0x9, 0x7f06bcad9878, 0x18, 0xc01727b7a0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0023df7d8, 0x9, 0x9, 0x58a03a0, 0xc0016f35c0, 0x0, 0x0, 0xc0015a69c0, 0xc004504d5f) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0023df7a0, 0xc00cd43038, 0xc00cd43038, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001881860) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5028 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1922 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002407830) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3660 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00011c3f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00011c3f0, 0xc0003a22a0, 0x1, 0x1, 0xc002031fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001474500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4823 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func2(0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1563 +0xb3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc0006db140) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4816 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x3a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1630) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3657 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc002a69c20, 0xc00142b320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3647 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc00049e270) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc00049e270, 0xc0003a22a0, 0x1, 0x1, 0xc001f17fb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001c99c00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 3524 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1(0x5922350, 0xc00203a000, 0xc000d48d80, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:66 +0x12f | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0012ca870, 0xc0012ca8a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 5224 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0013765a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 4814 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x39) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1610) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5495 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0xc000000006) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31040) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5491 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x100000000) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31000) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3509 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1(0x5922350, 0xc001304000, 0xc001f14790, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:66 +0x12f | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x1, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0012cadb0, 0xc0012caf90) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 5030 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0023cccd0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 3780 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x59223f8, 0xc0014fc150) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1339 +0x325 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0015de330, 0x0, 0x0, 0xc0030aa750) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4773 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0xe) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5027 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func3(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2206 +0x4d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002489460) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3912 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).RefreshLeases.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:898 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0034167b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4723 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2162 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc000ef29c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5026 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2162 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002407800) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4920 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/stats/automatic_stats.go:260 +0x18a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0020fdc60) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3910 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics.(*Registry).poll(0xc002008120, 0x5922350, 0xc002544380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics/statement_diagnostics.go:147 +0x2ba | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc002544380, 0x0, 0x0, 0xc000677400) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4881 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| crypto/tls.(*listener).Accept(0xc002407728, 0x40dbce0, 0xc00353ff50, 0x3d1ed60, 0x7646620) | |
| /usr/local/go/src/crypto/tls/tls.go:67 +0x37 | |
| net/http.(*Server).Serve(0xc0022a7b20, 0x58ef210, 0xc002407728, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2961 +0x285 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func4(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2309 +0x3f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002407740) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3909 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/stats/automatic_stats.go:260 +0x18a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc00254d440) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4880 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| net/http.(*Server).Serve(0xc0022a7c00, 0x5905590, 0xc0024076b0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2961 +0x285 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func3(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2296 +0x127 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00353f080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4812 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x36) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf15f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4879 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca2b520, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc003681918, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc003681900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc003681900, 0xc, 0x7f06bcad85b8, 0x10) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc002407680, 0x4295b8, 0xc, 0x3ec7fe0) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc002407680, 0xc000a174d0, 0x1d10b90, 0xc00250e800, 0x5922350) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cmux.(*cMux).Serve(0xc00261b580, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x96 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func2(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2278 +0x2f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc0024076f8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4878 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2261 +0x53 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002407698) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5281 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc0023dec40, 0x5922350, 0xc001f02b00, 0xc001f02b00, 0xc003b300c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc0023dec40, 0x5922350, 0xc001f02b00, 0x10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:276 +0xd4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x5922350, 0xc003665740) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:263 +0x6d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc003665740, 0x0, 0x0, 0xc002606a38) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5450 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca77148, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000b48998, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc000b48980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc000b48980, 0xc, 0x7f06bcad93c8, 0x10) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc00118dba8, 0x4295b8, 0xc, 0x3ec7fe0) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc00118dba8, 0xc00369b830, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cmux.(*cMux).Serve(0xc0009d2f00, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x96 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func2(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2278 +0x2f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc00118dc08) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3908 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc00254d420) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3540 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc001c770e0, 0xc002643140) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3424 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x5922350, 0xc001f02580) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:403 +0x51 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc001f02580, 0x0, 0x0, 0xc003a02678) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4748 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc003e25710) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25710, 0x0, 0x0, 0xc002585e80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4182 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc002405630) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 5318 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc0000e6380, 0xc002eefb00, 0x59223f8, 0xc0022f7cb0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 3418 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x59223f8, 0xc0026402a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x76 | |
| runtime/pprof.Do(0x5922388, 0xc0026402a0, 0xc0003a22a0, 0x1, 0x1, 0xc00165afb8) | |
| /usr/local/go/src/runtime/pprof/runtime.go:40 +0xcf | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000b5a600) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:174 +0x89 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:166 +0x1a5 | |
| goroutine 4876 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1(0x49b2640, 0xc000e46b90, 0xc00048a850, 0x59223f8, 0xc001ca6ea0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:247 +0xff | |
| created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:240 +0xd0 | |
| goroutine 3691 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.startPurgeOldSessions.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/purge_auth_session.go:65 +0x225 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00146f420) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4815 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x3d) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1620) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4747 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc003e256b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e256b0, 0x0, 0x0, 0xc002585e60) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3744 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/idxusage.(*LocalIndexUsageStats).startStatsIngestionLoop.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/idxusage/local_idx_usage_stats.go:408 +0x10b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0013149a8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4744 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001692f50, 0xc000ef3858, 0x5, 0x5, 0x8, 0x594ae18, 0x45663c) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001692f50, 0xc000ef3858, 0x5, 0x5, 0xc000283080, 0xc0021c1d10, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc004471020, 0xc000ef3858, 0x5, 0x5, 0xc003308978, 0xc003308940, 0xacfcdc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc004471020, 0xc000ef3858, 0x5, 0x5, 0x5, 0x1ac2110, 0xc000029e00, 0xc0033089cf) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00235f8c0, 0xc000ef3858, 0x5, 0x5, 0x0, 0xc0039a0ce0, 0xc02360fd40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc000ef3848, 0x7fffffff, 0x2, 0x2, 0x9, 0xa, 0xc02f274c70, 0xc001267a80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc000ef3848, 0xc00235f8c0, 0x0, 0x0, 0x7fffffff, 0xc003308c60, 0x59058c0, 0x7a02de8, 0xc003308b40, 0x485260, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc000ef3848, 0x7f06bca6db20, 0xc00044af90, 0xc00235f8c0, 0x0, 0x0, 0x41ae880, 0xc023642870, 0x7fffffff, 0xc003308c60, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc002513980, 0x41ae880, 0xc023642870, 0xc003308c60, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:957 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc002513980, 0x8, 0x8d) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:808 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00235f680, 0xc003308de0, 0xc003308db0, 0x50, 0x48) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:662 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00235f680, 0x41ae880, 0xc023642870, 0x0, 0xc001b9d300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:807 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc00447c320, 0x5922350, 0xc001f6fb40, 0xc000251380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:636 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0xc00444aea0, 0x5978d58, 0xc00447c320, 0xc001e85e00, 0x5922350, 0xc001f6fb40, 0xc000251380, 0x169bb531b373b8f7, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:324 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x5922350, 0xc001f6fb40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:339 +0x9c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc001f6fb40, 0x0, 0x0, 0xc004472800) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4745 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc003e255f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e255f0, 0x0, 0x0, 0xc002585e20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4746 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc003e25650) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25650, 0x0, 0x0, 0xc002585e40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4918 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).startSQLStatsFlushLoop.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/provider.go:98 +0x1c5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc002fb6480) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4826 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc000125db0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3764 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2162 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0030b42a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3765 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func3(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2206 +0x4d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc002fe5d80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3766 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1922 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0030b42d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3767 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*loopbackListener).Accept(0xc0008b49c0, 0x49b0428, 0xc0029bc1c0, 0x58f6560, 0xc0008b49c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/loopback.go:49 +0xb5 | |
| google.golang.org/grpc.(*Server).Serve(0xc0029bc1c0, 0x58f6560, 0xc0008b49c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:786 +0x27f | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func2(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1930 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0030b42e8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3768 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0030a8370) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 3769 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc002fcc580) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 3770 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.ConfigureGRPCGateway.func5(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1963 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0030b4348) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3925 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1(0x5922350, 0xc000a769c0, 0xc000e3e380, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:66 +0x12f | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc00093f710, 0xc00093f860) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 3772 [select, 38 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00353de60, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc001f996d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3773 [select, 38 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00149c240, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc001f996e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3774 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00149c360, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc001f996f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3775 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func1(0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:193 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc001b68918) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3776 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c192) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330efb0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3777 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c190) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330efc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3794 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c18d) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330efd0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3795 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c199) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330efe0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3796 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1b0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330eff0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3797 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c19d) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f000) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3798 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c188) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f010) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3799 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1bc) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f020) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3800 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c186) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f030) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3801 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1be) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f040) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3802 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c19b) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f050) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3803 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c187) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f060) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3804 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c18e) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f070) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3805 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1c4) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3806 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1b6) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f090) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3807 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c195) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f0a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3808 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c18f) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f0b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3809 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1ba) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f0c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3810 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1c3) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f0d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3811 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c19f) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f0e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3812 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1ab) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f0f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3813 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1b9) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f100) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3814 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1a8) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f110) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3815 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c18c) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f120) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3816 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c19c) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3817 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1a0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f140) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3818 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1c2) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f150) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3819 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1a1) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f160) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3820 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1a2) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f170) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3821 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c191) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f180) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3822 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1a7) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f190) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3823 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1ae) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3824 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1a5) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f1b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3825 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c18b) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f1c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3826 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c197) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f1d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3827 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c189) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f1e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3828 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1a3) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f1f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3829 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1b2) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f200) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3830 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1ad) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f210) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3831 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c19a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f220) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3832 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1a4) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f230) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3833 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1c1) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3834 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1a9) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f250) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3835 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c196) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f260) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3836 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1aa) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f270) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3837 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1bd) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f280) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3838 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1af) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f290) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3839 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1b3) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f2a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3840 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1b4) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f2b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3841 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c18a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f2c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3842 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c194) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f2d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3843 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1a6) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f2e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3844 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c198) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f2f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3845 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1ac) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f300) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3846 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c19e) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f310) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3847 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c185) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3848 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1b1) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f330) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3849 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1b7) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f340) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3850 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1c0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f350) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3851 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c193) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f360) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3852 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1b5) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f370) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3853 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1b8) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3854 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1bb) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f390) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3855 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00203a610, 0xc00000c1bf) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc00203a600) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0037c9900, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f3a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3856 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc0037c9990) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc0037c9988) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:212 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f3b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3857 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop(0xc00250e800, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:626 +0x232 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f3c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3858 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop(0xc00250e800, 0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:688 +0x10e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f3d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3859 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func2(0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1563 +0xb3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc001b68930) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3860 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc001b6b090) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3861 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc001b6b0e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3862 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc001b6b130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4751 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc003e25890) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25890, 0x0, 0x0, 0xc002585ee0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3864 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startLeaseRenewer.func1(0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1783 +0x2b0 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f430) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3440 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0014fcea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fcea0, 0x0, 0x0, 0xc003300040) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3441 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc0014fcf00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fcf00, 0x0, 0x0, 0xc003300060) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3867 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1(0x59223f8, 0xc0014fd320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1892 +0x2b5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd320, 0x0, 0x0, 0xc00330f440) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3868 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1(0x59223f8, 0xc00049e480) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_rebalancer.go:190 +0x16d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc00049e480, 0x0, 0x0, 0xc001b68978) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3869 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:678 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc003012760) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3870 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossiping.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:643 +0x1df | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc001b68990) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3871 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc0024005a0, 0xc00142bf80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 3872 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider.func1(0xc001e26070) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1126 +0xda | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1121 +0x24e | |
| goroutine 3873 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.periodicallyPersistHLCUpperBound(0xc000d72410, 0xc000639880, 0xc00330f590, 0x49b2640, 0xc002642300, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1088 +0x150 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startPersistingHLCUpperBound.func2(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1164 +0x69 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc00049e930) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3890 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports.(*Reporter).Start.func2(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports/reporter.go:152 +0x1cd | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc001b689d8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3891 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.startSampleEnvironment.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2561 +0x1c9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0020b2a80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3892 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:751 +0x43d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc0020b2af0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4906 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache.(*Cache).periodicallyRefreshProtectedtsCache(0xc0039a0e70, 0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache/cache.go:160 +0x239 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0040ff170) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5448 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:113 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc003756780) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4752 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc003e258f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e258f0, 0x0, 0x0, 0xc002585f00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5388 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000dfb040, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0060b8120, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc00487fc80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5393 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc000e70580) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5666 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0013e4000, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006408ae0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc005216c00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5464 [select]: | |
| net.(*pipe).read(0xc00249df00, 0xc001a78000, 0x773, 0x773, 0xc001a78005, 0x12, 0x76e) | |
| /usr/local/go/src/net/pipe.go:159 +0x185 | |
| net.(*pipe).Read(0xc00249df00, 0xc001a78000, 0x773, 0x773, 0x428e5b, 0xc0024a1e18, 0x18) | |
| /usr/local/go/src/net/pipe.go:142 +0x56 | |
| crypto/tls.(*atLeastReader).Read(0xc0024a1e18, 0xc001a78000, 0x773, 0x773, 0x76e, 0xc000382000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc002a93e78, 0x58a0580, 0xc0024a1e18, 0x426be5, 0x3da8720, 0x4119f40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc002a93c00, 0x7f0691997fc0, 0xc00249df00, 0x5, 0xc00249df00, 0x11) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc002a93c00, 0x0, 0x0, 0xa9c5be) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc002a93c00, 0xc005eb2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc005e99380, 0xc0001a69d8, 0x9, 0x9, 0x14, 0x7f06bcad8f18, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc005e99380, 0xc0001a69d8, 0x9, 0x9, 0x9, 0xa7fa45, 0xc00e6f6bdc, 0xc002b59780) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0001a69d8, 0x9, 0x9, 0x58a03a0, 0xc005e99380, 0x0, 0x0, 0xc00e6f6bd0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001a69a0, 0xc00e6f6bd0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc004dcde00, 0xc005c6b5f0, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc001f46fc0, 0x5981b58, 0xc004dcde00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc001f46fc0, 0x5981b58, 0xc004dcde00, 0x435fb83, 0x8) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5320 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:113 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc000a1a7c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5422 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc000dacb60, 0x5922350, 0xc003bc3600, 0xc003bc3600, 0xc002f77f90) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc000dacb60, 0x5922350, 0xc003bc3600, 0x10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:276 +0xd4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x5922350, 0xc003665740) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:263 +0x6d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc003665740, 0x0, 0x0, 0xc002e39200) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5418 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000c87220, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc004950480, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc005216300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5313 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001dc2d70, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc004b93080, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc002b050e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 4750 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc003e25830) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25830, 0x0, 0x0, 0xc002585ec0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5621 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00151fc20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 5595 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc003972000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 5347 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00342a9a0, 0x5922350, 0xc002c38840, 0xc002c38840, 0xc00387e6e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00342a9a0, 0x5922350, 0xc002c38840, 0x6) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:276 +0xd4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x5922350, 0xc000a76ac0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:263 +0x6d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc000a76ac0, 0x0, 0x0, 0xc0032d8060) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5346 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc00013a000, 0x6, 0xc0032853b0, 0x596be38, 0xc00387e6c0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:257 +0x670 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:143 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc00250e800, 0xc0032853b0, 0x596be38, 0xc00387e6c0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2530 +0x136 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc001e268c0, 0xc0032853b0, 0x596be38, 0xc00387e6c0, 0x10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:216 +0xce | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc00283cc00, 0xc0032853b0, 0x596be38, 0xc00387e6c0, 0xc00283cc00, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:1101 +0x54 | |
| github.com/cockroachdb/cockroach/pkg/roachpb._Internal_RangeFeed_Handler(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0016cfec0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8526 +0x113 | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0016cfec0, 0xc000e6de60, 0x49aa000, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:170 +0x5f7 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0016cfec0, 0x40c4c00, 0xc002c38680) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0016cfec0, 0xc000e6de60, 0xc002c38680, 0xc002c38680, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:186 +0x99 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0016cfec0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x1, 0x42ead40, 0xc00283cc00, 0x5966670, 0xc0016cfec0, 0xc000e6de60, 0xc002c38600, 0x40c4c00, 0xc002c38600) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb7 | |
| google.golang.org/grpc.chainStreamServerInterceptors.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0016cfec0, 0xc000e6de60, 0x49aa000, 0x59223f8, 0xc00011d8c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1401 +0xbd | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0029bc1c0, 0x5981b58, 0xc0054e1c80, 0xc0033237a0, 0xc0015df830, 0x765a660, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1549 +0x6d8 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0029bc1c0, 0x5981b58, 0xc0054e1c80, 0xc0033237a0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1622 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0006fb8c0, 0xc0029bc1c0, 0x5981b58, 0xc0054e1c80, 0xc0033237a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:941 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:939 +0x1fd | |
| goroutine 5476 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| google.golang.org/grpc.(*Server).Serve(0xc001f46fc0, 0x5905590, 0xc00118dd40, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:786 +0x27f | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2232 +0x3f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc003757c40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5373 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001d975e0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002def0e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc002a99380) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 4798 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc001f6fcd0, 0x2a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001f6fcc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc003cf5040, 0x59223f8, 0xc003e25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc003e25a40, 0x0, 0x0, 0xc003cf1510) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 3969 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| net/http.(*Server).Serve(0xc003e05a40, 0x5905590, 0xc000ef2870, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2961 +0x285 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startServeUI.func3(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2296 +0x127 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc002fb2870) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5282 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/idxusage.(*LocalIndexUsageStats).startStatsIngestionLoop.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/idxusage/local_idx_usage_stats.go:408 +0x10b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc000641080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5681 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046630, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0012dcf98, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0012dcf80, 0xc0022d2800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0012dcf80, 0xc0022d2800, 0x773, 0x773, 0xc0022d2805, 0xc001a80158, 0xc006900828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc005abd8f8, 0xc0022d2800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0068ede00, 0xc0022d2800, 0x773, 0x773, 0x428e5b, 0xc01e5f2468, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc01e5f2468, 0xc0022d2800, 0x773, 0x773, 0x76e, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0068ea978, 0x58a0580, 0xc01e5f2468, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0068ea700, 0x7f06919ac748, 0xc0068ede00, 0x5, 0xc0068ede00, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0068ea700, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0068ea700, 0xc006bd6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc006bc8780, 0xc000dad7d8, 0x9, 0x9, 0xc006900d28, 0x420ee5, 0xc005b61a40) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc006bc8780, 0xc000dad7d8, 0x9, 0x9, 0x9, 0xc01e5f2450, 0x481e60, 0xc005b61a40) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000dad7d8, 0x9, 0x9, 0x58a03a0, 0xc006bc8780, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000dad7a0, 0xc029163810, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc005216d80, 0xc006bc0f90, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc001f46fc0, 0x5981b58, 0xc005216d80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc001f46fc0, 0x5981b58, 0xc005216d80, 0xc006353ba0, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5668 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046e58, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001550618, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001550600, 0xc0022d2000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001550600, 0xc0022d2000, 0x773, 0x773, 0xc0022d2005, 0xc000190518, 0xc005e87828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00469ab50, 0xc0022d2000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0060b8a20, 0xc0022d2000, 0x773, 0x773, 0x428e5b, 0xc00fab8330, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc00fab8330, 0xc0022d2000, 0x773, 0x773, 0x76e, 0xc000aaa000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00191c278, 0x58a0580, 0xc00fab8330, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00191c000, 0x7f06919ac748, 0xc0060b8a20, 0x5, 0xc0060b8a20, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00191c000, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00191c000, 0xc006418000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc006408a80, 0xc000dacd58, 0x9, 0x9, 0xc005e87d28, 0x420ee5, 0xc005b60960) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc006408a80, 0xc000dacd58, 0x9, 0x9, 0x9, 0xc00fab8318, 0x481e60, 0xc005b60960) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000dacd58, 0x9, 0x9, 0x58a03a0, 0xc006408a80, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000dacd20, 0xc0288d0630, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc005216c00, 0xc005297b30, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc00201ddc0, 0x5981b58, 0xc005216c00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00201ddc0, 0x5981b58, 0xc005216c00, 0xc001f29010, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5438 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc002290000, 0xc001309220, 0xc00610c780, 0xf, 0xc005cdb080, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc0039efe00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc0039efe00, 0x0, 0x0, 0xc001309360) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5312 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca774e8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00249c318, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00249c300, 0xc005212000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00249c300, 0xc005212000, 0x8b5, 0x8b5, 0x4344de, 0x7f067ec7cd00, 0x20300a00000000) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000c62268, 0xc005212000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc00ddf7da0, 0xc005212000, 0x8b5, 0x8b5, 0x5167a5, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc002126278, 0x58a0580, 0xc00ddf7da0, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc002126000, 0x58a88a0, 0xc000c62268, 0x5, 0xc000c62268, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc002126000, 0x0, 0x0, 0xc0298183f0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc002126000, 0xc00501a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc004b92fc0, 0xc0001709d8, 0x9, 0x9, 0x7f068b6b18b8, 0x48, 0xc0291d9d98) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc004b92fc0, 0xc0001709d8, 0x9, 0x9, 0x9, 0xc029190001, 0xc003dced38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0001709d8, 0x9, 0x9, 0x58a03a0, 0xc004b92fc0, 0x0, 0xc000000000, 0xaba293, 0xc002b050e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001709a0, 0xc0291e7d50, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc002b050e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5667 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc005216c00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5323 [select]: | |
| net.(*pipe).read(0xc00249de80, 0xc00559c900, 0x8b5, 0x8b5, 0xc00559c905, 0x12, 0x8b0) | |
| /usr/local/go/src/net/pipe.go:159 +0x185 | |
| net.(*pipe).Read(0xc00249de80, 0xc00559c900, 0x8b5, 0x8b5, 0x428e5b, 0xc0024a1e00, 0x18) | |
| /usr/local/go/src/net/pipe.go:142 +0x56 | |
| crypto/tls.(*atLeastReader).Read(0xc0024a1e00, 0xc00559c900, 0x8b5, 0x8b5, 0x8b0, 0xc000382000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0032b1af8, 0x58a0580, 0xc0024a1e00, 0x426be5, 0x3da8720, 0x4119f40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0032b1880, 0x7f0691997fc0, 0xc00249de80, 0x5, 0xc00249de80, 0x11) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0032b1880, 0x0, 0x0, 0xa9c5be) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0032b1880, 0xc005f02000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc002b34e40, 0xc002465ed8, 0x9, 0x9, 0x7f068e4fa640, 0xc002139980, 0xc005e81cf8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc002b34e40, 0xc002465ed8, 0x9, 0x9, 0x9, 0x7f06bcad8f18, 0x18, 0xc0024a1de8) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc002465ed8, 0x9, 0x9, 0x58a03a0, 0xc002b34e40, 0x0, 0x0, 0xc001ea2a20, 0xc005e81d5f) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc002465ea0, 0xc00e6f6bb8, 0xc00e6f6bb8, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001b3bc20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5436 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc001309270) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 5419 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc005216300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5294 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func9(0x5922388, 0xc0000394c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:872 +0x215 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc002de5fb0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5293 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func8(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:849 +0x1d9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc000641188) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5292 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func7(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:817 +0x1ef | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc002558d20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5291 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).PeriodicallyRefreshSomeLeases.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1007 +0x2ec | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00279fb50) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5290 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).RefreshLeases.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:898 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002de5ef0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5289 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00249e820, 0x5922350, 0xc000903700, 0xc002de5e30, 0xc001311380, 0xc0013113e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xe7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00249e820, 0x5922350, 0xc000903700, 0xc002de5e30) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:226 +0x5f8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*Factory).RangeFeed.func1(0x5922350, 0xc000903700) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:137 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc000903700, 0x0, 0x0, 0xc000641140) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5288 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics.(*Registry).poll(0xc0020ad8c0, 0x5922350, 0xc000903580) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics/statement_diagnostics.go:147 +0x2ba | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc000903580, 0x0, 0x0, 0xc00279fa70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5287 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/stats/automatic_stats.go:260 +0x18a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc002558a20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5286 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002558a00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4923 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).RefreshLeases.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:898 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc004d5f6b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4922 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00061e4e0, 0x5922350, 0xc004a6ce40, 0xc004d5f5f0, 0xc004a69920, 0xc004a69980, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xe7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00061e4e0, 0x5922350, 0xc004a6ce40, 0xc004d5f5f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:226 +0x5f8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*Factory).RangeFeed.func1(0x5922350, 0xc004a6ce40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:137 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc004a6ce40, 0x0, 0x0, 0xc002fb6510) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4921 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics.(*Registry).poll(0xc001fe1620, 0x5922350, 0xc004a6cdc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics/statement_diagnostics.go:147 +0x2ba | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc004a6cdc0, 0x0, 0x0, 0xc004db82c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5497 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x5) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31060) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4915 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/idxusage.(*LocalIndexUsageStats).startStatsIngestionLoop.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/idxusage/local_idx_usage_stats.go:408 +0x10b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc002fb6468) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7336 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc003b4ed20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 5215 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier.(*Notifier).run(0xc000f26c00, 0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier/notifier.go:136 +0xea | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00279f980) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5075 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x59223f8, 0xc0033b42a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1339 +0x325 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc0033b5710, 0x0, 0x0, 0xc0053b9ef0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4670 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc00208c160, 0x59223f8, 0xc0037786c0, 0x10, 0xc0036ff938, 0xc000e3e380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:242 +0x40c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1(0x59223f8, 0xc0037786c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:197 +0x59 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0037786c0, 0x0, 0x0, 0xc0027a4c00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4718 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002229c20, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0043e5e00, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc001881860) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 4719 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0009f8d20, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0043e5e60, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc003b50300) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 4743 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc002363180, 0xc00235f680, 0x59223f8, 0xc004470f60) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 4730 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 | |
| google.golang.org/grpc.(*Server).Serve(0xc0022b7c00, 0x5905590, 0xc000ef29d8, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:786 +0x27f | |
| github.com/cockroachdb/cockroach/pkg/server.StartListenRPCAndSQL.func5.1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2232 +0x3f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0025852c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5106 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc001b6b9a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 4185 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001ace780) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 5214 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoop(0xc000ffdc20, 0x5922350, 0xc0038baac0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:214 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00279f970) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5213 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage.(*Storage).deleteSessionsLoop(0xc0021c56b0, 0x59223f8, 0xc0038ba9c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage/slstorage.go:317 +0x1fa | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc00279f900) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4824 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc000125c70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4825 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc000125cc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5285 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).startSQLStatsFlushLoop.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/provider.go:98 +0x1c5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc000641098) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5474 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc005ac4840) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 4740 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc0022478c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5485 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x59223f8, 0xc005efe570) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1339 +0x325 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc002207fb0, 0x0, 0x0, 0xc003db4030) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4741 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000aef440, 0xc001692280, 0xc003520980, 0xf, 0xc00444a480, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc00063f4c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc00063f4c0, 0x0, 0x0, 0xc001692410) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4833 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:678 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0005d2100) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4834 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossiping.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:643 +0x1df | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0006db1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4835 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc003cf5cc0, 0xc00457e7e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 4836 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider.func1(0xc0009f10a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1126 +0xda | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1121 +0x24e | |
| goroutine 4837 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.periodicallyPersistHLCUpperBound(0xc00122ea50, 0xc001c352d0, 0xc003cf1850, 0x49b2640, 0xc000028cc0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1088 +0x150 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startPersistingHLCUpperBound.func2(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1164 +0x69 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0045ac030) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4838 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports.(*Reporter).Start.func2(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports/reporter.go:152 +0x1cd | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0006db1e8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4839 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.startSampleEnvironment.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2561 +0x1c9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc003e72770) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4840 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:751 +0x43d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc003e727e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4875 [select, 39 minutes]: | |
| database/sql.(*DB).connectionOpener(0xc002277e10, 0x5922350, 0xc00261b440) | |
| /usr/local/go/src/database/sql/sql.go:1133 +0xb5 | |
| created by database/sql.OpenDB | |
| /usr/local/go/src/database/sql/sql.go:740 +0x12a | |
| goroutine 4993 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca77b40, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc002318218, 0x72, 0x4700, 0x4792, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc002318200, 0xc0048e3000, 0x4792, 0x4792, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc002318200, 0xc0048e3000, 0x4792, 0x4792, 0x478d, 0xc0048e3000, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00172c280, 0xc0048e3000, 0x4792, 0x4792, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc017334d20, 0xc0048e3000, 0x4792, 0x4792, 0x478d, 0xc0000a0800, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0037da978, 0x58a0580, 0xc017334d20, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0037da700, 0x58a88a0, 0xc00172c280, 0x5, 0xc00172c280, 0x9b) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0037da700, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0037da700, 0xc00363e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc001d2b440, 0xc004e03378, 0x9, 0x9, 0x4221c5, 0xc0013bfda0, 0xc0023f7ce8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc001d2b440, 0xc004e03378, 0x9, 0x9, 0x9, 0xc00229b700, 0x5318d9, 0xc001f4abc8) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc004e03378, 0x9, 0x9, 0x58a03a0, 0xc001d2b440, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc004e03340, 0xc003725560, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00367fa40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 4846 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000978960, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0045e1d40, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc003cf3b00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 4847 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc003cf3b00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 4848 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca2b9a8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00107af18, 0x72, 0xe700, 0xe777, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00107af00, 0xc00882e000, 0xe777, 0xe777, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00107af00, 0xc00882e000, 0xe777, 0xe777, 0xc00882e005, 0xc000f508d8, 0xc004945828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc0005ce9a0, 0xc00882e000, 0xe777, 0xe777, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc003db6ba0, 0xc00882e000, 0xe777, 0xe777, 0x428e5b, 0xc00fab8690, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc00fab8690, 0xc00882e000, 0xe777, 0xe777, 0xe772, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc003d86cf8, 0x58a0580, 0xc00fab8690, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc003d86a80, 0x7f06919ac748, 0xc003db6ba0, 0x5, 0xc003db6ba0, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc003d86a80, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc003d86a80, 0xc004862000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0045e1ce0, 0xc003e05ed8, 0x9, 0x9, 0xc004945d28, 0x420ee5, 0xc00457ede0) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0045e1ce0, 0xc003e05ed8, 0x9, 0x9, 0x9, 0xc00fab8678, 0x481e60, 0xc00457ede0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc003e05ed8, 0x9, 0x9, 0x58a03a0, 0xc0045e1ce0, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc003e05ea0, 0xc0288d0990, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc003cf3b00, 0xc0045ad0b0, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0029bc1c0, 0x5981b58, 0xc003cf3b00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0029bc1c0, 0x5981b58, 0xc003cf3b00, 0xc00375aec0, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 4703 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000cf5220, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc004b89e00, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc003dbe1e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 4884 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000978fa0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 4885 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc003e1b8c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 4886 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000aef440, 0xc000978f50, 0xc001d726c0, 0xf, 0xc00457f0e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc00063f4c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc00063f4c0, 0x0, 0x0, 0xc000979090) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4701 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc003dbe1e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 4905 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func1(0x59223f8, 0xc0045ac1e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:754 +0xef | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0045ac1e0, 0x0, 0x0, 0xc002fb6270) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5433 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001308eb0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0061077a0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc003884f00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 4924 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).PeriodicallyRefreshSomeLeases.func1(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1007 +0x2ec | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc004db8320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4925 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func7(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:817 +0x1ef | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0020fddc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4926 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func8(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:849 +0x1d9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc002fb6540) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4927 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func9(0x5922388, 0xc004a6d480) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:872 +0x215 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc004d5f770) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5084 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Sender).Run.func2(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:253 +0x1a9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc0030138a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4929 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc004dd83d0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc004dd83c8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc004dd83c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc004dd83c0, 0x5922350, 0xc004a6d300, 0xc004dd2480, 0xc004a6d300) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc001e7f200, 0x5922350, 0xc004dd8360, 0xc004d79000, 0x1, 0x8, 0xc004d79008, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:83 +0x3ca | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc000e6de00, 0x5922350, 0xc004a6ce40, 0xc004d79000, 0x1, 0x8, 0xc004d79008, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:69 +0xb4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1(0x5922350, 0xc004a6ce40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:220 +0xe3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc004a6ce40, 0x0, 0x0, 0xc004dd8330) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4946 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1(0x5922350, 0xc004a6d300, 0xc000d48d80, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:66 +0x12f | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc004dd83c0, 0xc004dd83f0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4948 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0012cf180, 0xc002fb6bf8, 0x5, 0x5, 0xffc03c, 0xc006c6bbb0, 0xc012f55e0c) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0012cf180, 0xc002fb6bf8, 0x5, 0x5, 0xc0032cf2c0, 0xc004db63d8, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc004dd8660, 0xc002fb6bf8, 0x5, 0x5, 0x0, 0xc0032cf320, 0xacfcdc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc004dd8660, 0xc002fb6bf8, 0x5, 0x5, 0x5, 0xc0032cf390, 0x426f05, 0x3da3b20) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc004dd0c60, 0xc002fb6bf8, 0x5, 0x5, 0xc012f55e00, 0x16, 0x200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc002fb6be8, 0x7fffffff, 0x19e2de2, 0x7a02de8, 0xc012f55e00, 0x8, 0x200, 0x4252220) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc002fb6be8, 0xc004dd0c60, 0x0, 0x0, 0x7fffffff, 0xc0032cf640, 0x59058c0, 0x7a02de8, 0xc028113980, 0x5a3df06e790, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc002fb6be8, 0x7f06bca6db20, 0xc00044af90, 0xc004dd0c60, 0x0, 0x0, 0x4252220, 0xc00920eb70, 0x7fffffff, 0xc0032cf640, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc000f23380, 0x4252220, 0xc00920eb70, 0xc0032cf640, 0x45663c, 0x770000c0032cf6a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:957 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000f23380, 0xc0032cf6e8, 0x421105) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:808 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc004dd0a20, 0xc0032cf7c0, 0xc0032cf790, 0x18, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:662 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc004dd0a20, 0x4252220, 0xc00920eb70, 0x2, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:807 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc004db86d0, 0xc0032cf8c8, 0x0, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8375 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc001e7f200, 0x5922350, 0xc004a6d300, 0xc004d79000, 0x1, 0x8, 0xc004d79008, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:270 +0x911 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc001e7f200, 0x5922350, 0xc004a6d300, 0xc004db5e30, 0x0, 0xc004dd2420, 0xc004a69920, 0x1150273, 0x5922388) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:151 +0x34c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1(0x5922350, 0xc004a6d300, 0xc000d48d80, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:70 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc004dd83c0, 0xc004dd8480) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4949 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc001c4b880, 0xc004dd0a20, 0x59223f8, 0xc004dd85a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 4950 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc00013a000, 0x6, 0xc004006a50, 0x596be38, 0xc004db87f0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:257 +0x670 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:143 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc00250e800, 0xc004006a50, 0x596be38, 0xc004db87f0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2530 +0x136 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc001e268c0, 0xc004006a50, 0x596be38, 0xc004db87f0, 0x10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:216 +0xce | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc00283cc00, 0xc004006a50, 0x596be38, 0xc004db87f0, 0xc00283cc00, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:1101 +0x54 | |
| github.com/cockroachdb/cockroach/pkg/roachpb._Internal_RangeFeed_Handler(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004dea000, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8526 +0x113 | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004dea000, 0xc002fb6d50, 0x49aa000, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:170 +0x5f7 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004dea000, 0x40c4c00, 0xc004a6d740) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004dea000, 0xc002fb6d50, 0xc004a6d740, 0xc004a6d740, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:186 +0x99 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004dea000, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x1, 0x42ead40, 0xc00283cc00, 0x5966670, 0xc004dea000, 0xc002fb6d50, 0xc004a6d700, 0x40c4c00, 0xc004a6d700) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb7 | |
| google.golang.org/grpc.chainStreamServerInterceptors.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc004dea000, 0xc002fb6d50, 0x49aa000, 0x59223f8, 0xc004dd8930) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1401 +0xbd | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0029bc1c0, 0x5981b58, 0xc003cf3b00, 0xc004dd0d80, 0xc0015df830, 0x765a660, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1549 +0x6d8 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0029bc1c0, 0x5981b58, 0xc003cf3b00, 0xc004dd0d80, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1622 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00456efc0, 0xc0029bc1c0, 0x5981b58, 0xc003cf3b00, 0xc004dd0d80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:941 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:939 +0x1fd | |
| goroutine 4894 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Sender).Run.func2(0x59223f8, 0xc0012ca8d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:253 +0x1a9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc0024886a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5202 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022ae8c0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001d2b500, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00367fa40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 4896 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1(0x59223f8, 0xc004849740) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:151 +0x2ca | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x59223f8, 0xc0012ca8d0, 0x0, 0x0, 0xc002eb1158) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 4951 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc003c176c0, 0x5922350, 0xc004a6d880, 0xc004a6d880, 0xc004db8810) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc003c176c0, 0x5922350, 0xc004a6d880, 0x6) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:276 +0xd4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x5922350, 0xc000a76ac0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:263 +0x6d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc000a76ac0, 0x0, 0x0, 0xc002fb6de0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5211 [select, 9 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startSystemLogsGC.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server_systemlog_gc.go:177 +0x545 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc000640780) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5065 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc00124cf00, 0xc00263b1e0, 0x5, 0x5, 0x8, 0x594ae18, 0x45663c) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00124cf00, 0xc00263b1e0, 0x5, 0x5, 0xc001dd2000, 0xc00374c060, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc003698900, 0xc00263b1e0, 0x5, 0x5, 0xc004de6978, 0xc004de6940, 0xacfcdc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc003698900, 0xc00263b1e0, 0x5, 0x5, 0x5, 0x1ac2110, 0xc00142b080, 0xc004de69cf) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc002eeeb40, 0xc00263b1e0, 0x5, 0x5, 0x0, 0xc0021c5470, 0xc01cae2d80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc00263b1d0, 0x7fffffff, 0x2, 0x2, 0x9, 0xa, 0xc02aef10e0, 0xc0024a8120) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc00263b1d0, 0xc002eeeb40, 0x0, 0x0, 0x7fffffff, 0xc004de6c60, 0x59058c0, 0x7a02de8, 0xc004de6b40, 0x485260, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc00263b1d0, 0x7f06bca6db20, 0xc00044af90, 0xc002eeeb40, 0x0, 0x0, 0x41ae880, 0xc01e60de50, 0x7fffffff, 0xc004de6c60, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc001937880, 0x41ae880, 0xc01e60de50, 0xc004de6c60, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:957 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001937880, 0x8, 0x8d) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:808 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc002eee900, 0xc004de6de0, 0xc004de6db0, 0x50, 0x48) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:662 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc002eee900, 0x41ae880, 0xc01e60de50, 0x0, 0xc00014b7b0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:807 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc00244de70, 0x5922350, 0xc002544080, 0xc0015069c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:636 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0xc001a48660, 0x5978d58, 0xc00244de70, 0xc00249b2c0, 0x5922350, 0xc002544080, 0xc0015069c0, 0x0, 0x79b3520) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:324 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x5922350, 0xc002544080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:339 +0x9c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc002544080, 0x0, 0x0, 0xc001f4acc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5066 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc003298780, 0x5978db0, 0xc00089a9b0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:190 +0x6c5 | |
| github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x420a960, 0xc003298780, 0x5966670, 0xc003c01c80, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:660 +0xad | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x420a960, 0xc003298780, 0x5966670, 0xc003c01c80, 0xc00263b380, 0x49a94f8, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:170 +0x5f7 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x420a960, 0xc003298780, 0x5966670, 0xc003c01c80, 0x40c4c00, 0xc001f4af40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x420a960, 0xc003298780, 0x5966670, 0xc003c01c80, 0xc00263b380, 0xc001f4af40, 0xc001f4af40, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:186 +0x99 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x420a960, 0xc003298780, 0x5966670, 0xc003c01c80, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x1, 0x420a960, 0xc003298780, 0x5966670, 0xc003c01c80, 0xc00263b380, 0xc001f4af00, 0x40c4c00, 0xc001f4af00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb7 | |
| google.golang.org/grpc.chainStreamServerInterceptors.func1(0x420a960, 0xc003298780, 0x5966670, 0xc003c01c80, 0xc00263b380, 0x49a94f8, 0x59223f8, 0xc003698d20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1401 +0xbd | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0029bc1c0, 0x5981b58, 0xc00237b500, 0xc002eeec60, 0xc0015de750, 0x764e240, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1549 +0x6d8 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0029bc1c0, 0x5981b58, 0xc00237b500, 0xc002eeec60, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1622 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00011efd0, 0xc0029bc1c0, 0x5981b58, 0xc00237b500, 0xc002eeec60) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:941 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:939 +0x1fd | |
| goroutine 4960 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000ed62d0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00065f680, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc003b4e000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 4961 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000ed66e0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00372a1e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc002a98180) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 4994 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002a98180) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 4995 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca2b608, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000f23a98, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000f23a80, 0xc002f06000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000f23a80, 0xc002f06000, 0x773, 0x773, 0xc002f06005, 0xc0024bc298, 0xc003dcd828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc004033910, 0xc002f06000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc004dfe900, 0xc002f06000, 0x773, 0x773, 0x428e5b, 0xc00ddf7e30, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc00ddf7e30, 0xc002f06000, 0x773, 0x773, 0x76e, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc004df8cf8, 0x58a0580, 0xc00ddf7e30, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc004df8a80, 0x7f06919ac748, 0xc004dfe900, 0x5, 0xc004dfe900, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc004df8a80, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc004df8a80, 0xc004158000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc00372a180, 0xc003c161f8, 0x9, 0x9, 0xc003dcdd28, 0x420ee5, 0xc004ee4780) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc00372a180, 0xc003c161f8, 0x9, 0x9, 0x9, 0xc00ddf7e18, 0x481e60, 0xc004ee4780) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc003c161f8, 0x9, 0x9, 0x58a03a0, 0xc00372a180, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc003c161c0, 0xc0291e7d70, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002a98180, 0xc004dfced0, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0022b7c00, 0x5981b58, 0xc002a98180) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0022b7c00, 0x5981b58, 0xc002a98180, 0xc004d79550, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5042 [select]: | |
| net.(*pipe).read(0xc003681e80, 0xc004b8d200, 0x8b5, 0x8b5, 0xc004b8d205, 0x12, 0x8b0) | |
| /usr/local/go/src/net/pipe.go:159 +0x185 | |
| net.(*pipe).Read(0xc003681e80, 0xc004b8d200, 0x8b5, 0x8b5, 0x428e5b, 0xc01a6194d0, 0x18) | |
| /usr/local/go/src/net/pipe.go:142 +0x56 | |
| crypto/tls.(*atLeastReader).Read(0xc01a6194d0, 0xc004b8d200, 0x8b5, 0x8b5, 0x8b0, 0xc000604000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00508d778, 0x58a0580, 0xc01a6194d0, 0x426be5, 0x3da8720, 0x4119f40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00508d500, 0x7f0691997fc0, 0xc003681e80, 0x5, 0xc003681e80, 0x11) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00508d500, 0x0, 0x0, 0xa9c5be) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00508d500, 0xc002696000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc00487a180, 0xc00309c2d8, 0x9, 0x9, 0x14, 0xc00262e000, 0xc0023fbcf8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc00487a180, 0xc00309c2d8, 0x9, 0x9, 0x9, 0x7f06bcad8108, 0x18, 0xc01a6194b8) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00309c2d8, 0x9, 0x9, 0x58a03a0, 0xc00487a180, 0x0, 0x0, 0xc002b00ae0, 0xc0023fbd5f) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00309c2a0, 0xc00bb1a690, 0xc00bb1a690, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc002e46960) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5043 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0025361e0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00487a600, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc002e46960) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 5044 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002536140, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00487a7e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc002b6fe00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5045 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002b6fe00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5046 [select]: | |
| net.(*pipe).read(0xc003681f00, 0xc004afc800, 0x773, 0x773, 0xc004afc805, 0x12, 0x76e) | |
| /usr/local/go/src/net/pipe.go:159 +0x185 | |
| net.(*pipe).Read(0xc003681f00, 0xc004afc800, 0x773, 0x773, 0x428e5b, 0xc01a6194e8, 0x18) | |
| /usr/local/go/src/net/pipe.go:142 +0x56 | |
| crypto/tls.(*atLeastReader).Read(0xc01a6194e8, 0xc004afc800, 0x773, 0x773, 0x76e, 0xc000604000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc002363e78, 0x58a0580, 0xc01a6194e8, 0x426be5, 0x3da8720, 0x4119f40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc002363c00, 0x7f0691997fc0, 0xc003681f00, 0x5, 0xc003681f00, 0x11) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc002363c00, 0x0, 0x0, 0xa9c5be) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc002363c00, 0xc00261c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc00487a120, 0xc00309c1f8, 0x9, 0x9, 0x14, 0x7f06bcad8108, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc00487a120, 0xc00309c1f8, 0x9, 0x9, 0x9, 0xa7fa45, 0xc00bb1a6b4, 0xc0052b1f20) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00309c1f8, 0x9, 0x9, 0x58a03a0, 0xc00487a120, 0x0, 0x0, 0xc00bb1a6a8, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00309c1c0, 0xc00bb1a6a8, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002b6fe00, 0xc0037876e0, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc00201ddc0, 0x5981b58, 0xc002b6fe00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00201ddc0, 0x5981b58, 0xc002b6fe00, 0x435fb83, 0x8) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 4897 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc00249b2c0, 0x5922350, 0xc002544080, 0xc0015069c0, 0x5978d58, 0xc00244de70, 0xc00256de80, 0xc0027fe020, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:371 +0x3bf | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1(0x5922350, 0xc002544080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:125 +0x44e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc002544080, 0x0, 0x0, 0xc0025440c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5314 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc0022f7ae0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc0022f7ad8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0022f7ad0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc0022f7ad0, 0x5922350, 0xc003774c80, 0xc001e9b9e0, 0xc003774c80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc000dffb00, 0x5922350, 0xc0022f7a70, 0xc003521f10, 0x1, 0x8, 0xc003521f18, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:83 +0x3ca | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc0026f7470, 0x5922350, 0xc000903700, 0xc003521f10, 0x1, 0x8, 0xc003521f18, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:69 +0xb4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1(0x5922350, 0xc000903700) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:220 +0xe3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc000903700, 0x0, 0x0, 0xc0022f7a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5091 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc003784000, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0026949a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5092 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc003784120, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0026949b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5093 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc003784240, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0026949c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5094 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func1(0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:193 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0030b46d8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5095 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057c1) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0026949f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5096 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057c2) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694ab0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5097 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057c3) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694ac0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5098 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057c4) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694ad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5099 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057c5) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694ae0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5100 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057c6) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694af0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5101 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057c7) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694b00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5102 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057c8) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694b10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5103 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057c9) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694b20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5104 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057ca) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694b30) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5105 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057cb) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694b40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5122 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057cc) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694b50) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5123 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057cd) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694b60) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5124 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057ce) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694cc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5125 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057cf) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694cd0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5126 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057d0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694ce0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5127 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057d1) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694cf0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5128 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057d2) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694d00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5129 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057d3) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694d10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5130 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057d4) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694d60) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5131 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057d5) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694e10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5132 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057d6) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694e20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5133 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057d7) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694e30) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5134 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057d8) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694e40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5135 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057d9) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694e50) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5136 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057da) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694e70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5137 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057db) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694e80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5138 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057dc) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694e90) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5139 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057dd) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5140 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057de) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694eb0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5141 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057df) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694ec0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5142 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057e0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694f60) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5143 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057e1) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694f70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5144 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057e2) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694f80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5145 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057e3) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694f90) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5146 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057a4) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694fa0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5147 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057a5) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694fb0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5148 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057a6) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694fc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5149 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057a7) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694fd0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5150 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057c0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694fe0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5151 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057a9) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002694ff0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5152 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057aa) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695000) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5153 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057ab) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695010) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5154 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057ac) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695090) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5155 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057ad) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0026950a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5156 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057ae) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0026950b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5157 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057af) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0026950c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5158 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057b0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0026950d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5159 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057b1) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0026950e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5160 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057b2) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0026950f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5161 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057b3) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695100) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5162 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057b4) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695110) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5163 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057b5) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695120) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5164 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057b6) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5165 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057b7) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695140) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5166 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057b8) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0026951c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5167 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057b9) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0026951d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5168 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057ba) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0026951e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5169 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057bb) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0026951f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5170 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057bc) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695200) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5171 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057bd) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695210) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5172 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057a8) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695220) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5173 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057be) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695230) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5174 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0025442d0, 0xc0000057bf) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0025442c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0021ad900, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5175 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc0021ad990) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc0021ad988) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:212 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695250) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5176 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop(0xc000cd3800, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:626 +0x232 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695270) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5177 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop(0xc000cd3800, 0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:688 +0x10e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695310) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5178 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func2(0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1563 +0xb3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc0030b46f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5179 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc002274690) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5180 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0022746e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5181 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc002274730) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5225 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc002fcd080) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5183 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startLeaseRenewer.func1(0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1783 +0x2b0 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695580) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5078 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00250b9f0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0013bfb60, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc00237b500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5446 [select, 39 minutes]: | |
| database/sql.(*DB).connectionOpener(0xc00257e750, 0x5922350, 0xc0009d27c0) | |
| /usr/local/go/src/database/sql/sql.go:1133 +0xb5 | |
| created by database/sql.OpenDB | |
| /usr/local/go/src/database/sql/sql.go:740 +0x12a | |
| goroutine 5186 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1(0x59223f8, 0xc002270450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1892 +0x2b5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270450, 0x0, 0x0, 0xc002695590) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5187 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1(0x59223f8, 0xc002270750) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_rebalancer.go:190 +0x16d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270750, 0x0, 0x0, 0xc0030b4738) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5188 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:678 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc003400c20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5189 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossiping.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:643 +0x1df | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc0030b4750) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5190 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc000a3f220, 0xc000f25a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 5191 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider.func1(0xc0038d7c00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1126 +0xda | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1121 +0x24e | |
| goroutine 5192 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.periodicallyPersistHLCUpperBound(0xc000e46b90, 0xc000639960, 0xc0026957c0, 0x49b2640, 0xc00142a7e0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1088 +0x150 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startPersistingHLCUpperBound.func2(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1164 +0x69 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc002270ab0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5193 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports.(*Reporter).Start.func2(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports/reporter.go:152 +0x1cd | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc0030b4798) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5194 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.startSampleEnvironment.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2561 +0x1c9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc0002456c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5195 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:751 +0x43d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc000245730) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5208 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func1(0x59223f8, 0xc002270c60) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:754 +0xef | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc002270c60, 0x0, 0x0, 0xc000640750) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5683 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001434190, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0061578c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc00219f980) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5315 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1(0x5922350, 0xc003774c80, 0xc00256de80, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:66 +0x12f | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0022f7ad0, 0xc0022f7b00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 5237 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca77970, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0023a6318, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0023a6300, 0xc003362000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0023a6300, 0xc003362000, 0x8b5, 0x8b5, 0x8b0, 0xc003362000, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00172c298, 0xc003362000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc01e5f2378, 0xc003362000, 0x8b5, 0x8b5, 0x5167a5, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0037dacf8, 0x58a0580, 0xc01e5f2378, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0037daa80, 0x58a88a0, 0xc00172c298, 0x5, 0xc00172c298, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0037daa80, 0x0, 0x0, 0xc029e86630) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0037daa80, 0xc003c1e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc00204aae0, 0xc001280118, 0x9, 0x9, 0x7f068b6eb6d0, 0x48, 0xc02916e598) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc00204aae0, 0xc001280118, 0x9, 0x9, 0x9, 0xc028d20001, 0xc002e0bd38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc001280118, 0x9, 0x9, 0x58a03a0, 0xc00204aae0, 0x0, 0xc000000000, 0xaba293, 0xc00090a1e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0012800e0, 0xc0291636e0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00090a1e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5238 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022756d0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00204aba0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00090a1e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 5048 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002536910, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00204acc0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc00262e600) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5049 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00262e600) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5050 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca77888, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc003563118, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc003563100, 0xc004afd000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc003563100, 0xc004afd000, 0x773, 0x773, 0xc004afd005, 0xc002466018, 0xc002db2828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00172c2a8, 0xc004afd000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc001cbd8c0, 0xc004afd000, 0x773, 0x773, 0x428e5b, 0xc01e5f23c0, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc01e5f23c0, 0xc004afd000, 0x773, 0x773, 0x76e, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0037db078, 0x58a0580, 0xc01e5f23c0, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0037dae00, 0x7f06919ac748, 0xc001cbd8c0, 0x5, 0xc001cbd8c0, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0037dae00, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0037dae00, 0xc003c32000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc000efe0c0, 0xc00309c738, 0x9, 0x9, 0xc002db2d28, 0x420ee5, 0xc002b01020) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc000efe0c0, 0xc00309c738, 0x9, 0x9, 0x9, 0xc01e5f23a8, 0x481e60, 0xc002b01020) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00309c738, 0x9, 0x9, 0x58a03a0, 0xc000efe0c0, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00309c700, 0xc0291636f0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00262e600, 0xc003787f50, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0022b7c00, 0x5981b58, 0xc00262e600) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0022b7c00, 0x5981b58, 0xc00262e600, 0xc0005d4bf0, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5317 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0000d27d0, 0xc003a03288, 0x5, 0x5, 0xffc03c, 0xc001b6d230, 0xc0126a1a0c) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0000d27d0, 0xc003a03288, 0x5, 0x5, 0xc0019332c0, 0xc00374c5d0, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0022f7d70, 0xc003a03288, 0x5, 0x5, 0x0, 0xc001933320, 0xacfcdc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc0022f7d70, 0xc003a03288, 0x5, 0x5, 0x5, 0xc001933390, 0x426f05, 0x3da3b20) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc002eefd40, 0xc003a03288, 0x5, 0x5, 0xc0126a1a00, 0x16, 0x200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc003a03278, 0x7fffffff, 0x19e2de2, 0x7a02de8, 0xc0126a1a00, 0x8, 0x200, 0x4252220) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc003a03278, 0xc002eefd40, 0x0, 0x0, 0x7fffffff, 0xc001933640, 0x59058c0, 0x7a02de8, 0xc01ea85560, 0x5a3df072a76, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc003a03278, 0x7f06bca6db20, 0xc00044af90, 0xc002eefd40, 0x0, 0x0, 0x4252220, 0xc01d9ab848, 0x7fffffff, 0xc001933640, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc0023a6700, 0x4252220, 0xc01d9ab848, 0xc001933640, 0x45663c, 0xc0019336a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:957 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0023a6700, 0xc0019336e8, 0x421105) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:808 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc002eefb00, 0xc0019337c0, 0xc001933790, 0x18, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:662 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc002eefb00, 0x4252220, 0xc01d9ab848, 0x2, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:807 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc000677a50, 0xc0019338c8, 0x0, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8375 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc000dffb00, 0x5922350, 0xc003774c80, 0xc003521f10, 0x1, 0x8, 0xc003521f18, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:270 +0x911 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc000dffb00, 0x5922350, 0xc003774c80, 0xc00396df10, 0xc00142b000, 0xc001e9b980, 0xc001311380, 0x1150273, 0x59223f8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:151 +0x34c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1(0x5922350, 0xc003774c80, 0xc00256de80, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:70 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0022f7ad0, 0xc0022f7b90) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 5241 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000150000, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00204b9e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc0054e1c80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5242 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0054e1c80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5243 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca777a0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc002257818, 0x72, 0xe500, 0xe512, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc002257800, 0xc00575a000, 0xe512, 0xe512, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc002257800, 0xc00575a000, 0xe512, 0xe512, 0xc00575a005, 0xc00226e018, 0xc0023f8828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc0005cf0c8, 0xc00575a000, 0xe512, 0xe512, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc001d2b560, 0xc00575a000, 0xe512, 0xe512, 0x428e5b, 0xc00ddf7f80, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc00ddf7f80, 0xc00575a000, 0xe512, 0xe512, 0xe50d, 0xc000280400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00267d078, 0x58a0580, 0xc00ddf7f80, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00267ce00, 0x7f06919ac748, 0xc001d2b560, 0x5, 0xc001d2b560, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00267ce00, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00267ce00, 0xc0041b8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc00204b980, 0xc0012802d8, 0x9, 0x9, 0xc0023f8d28, 0x420ee5, 0xc003fe4600) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc00204b980, 0xc0012802d8, 0x9, 0x9, 0x9, 0xc00ddf7f68, 0x481e60, 0xc003fe4600) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0012802d8, 0x9, 0x9, 0x58a03a0, 0xc00204b980, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0012802a0, 0xc0291e7ec0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0054e1c80, 0xc0024c7380, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0029bc1c0, 0x5981b58, 0xc0054e1c80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0029bc1c0, 0x5981b58, 0xc0054e1c80, 0xc000f16b50, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5251 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0025379a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 5252 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc0013dcb00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5255 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001292000, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001d2b620, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc00262fe00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5256 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00262fe00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5247 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca776b8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc003563c98, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc003563c80, 0xc003362900, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc003563c80, 0xc003362900, 0x8b5, 0x8b5, 0x8b0, 0xc003362900, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00172c340, 0xc003362900, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc017fa29a8, 0xc003362900, 0x8b5, 0x8b5, 0x8b0, 0xc002310400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0037dbe78, 0x58a0580, 0xc017fa29a8, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0037dbc00, 0x58a88a0, 0xc00172c340, 0x5, 0xc00172c340, 0xc003bdd960) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0037dbc00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0037dbc00, 0xc004344000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0022db500, 0xc001280818, 0x9, 0x9, 0x0, 0xc003bddc98, 0x58e6b70) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0022db500, 0xc001280818, 0x9, 0x9, 0x9, 0xc0000f7558, 0x9e7e783d60483a01, 0xc0301df1d0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc001280818, 0x9, 0x9, 0x58a03a0, 0xc0022db500, 0x0, 0x0, 0xc01bb48ed8, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0012807e0, 0xc0301df170, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00090a3c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5248 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000150960, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0022db620, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00090a3c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 5257 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca775d0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000c09d18, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000c09d00, 0xc004afd800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000c09d00, 0xc004afd800, 0x773, 0x773, 0xc004afd805, 0xc002466f18, 0xc002db4828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc0003a41a0, 0xc004afd800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc000eff920, 0xc004afd800, 0x773, 0x773, 0x428e5b, 0xc00c47de18, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc00c47de18, 0xc004afd800, 0x773, 0x773, 0xc007e32028, 0xc000280400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00508de78, 0x58a0580, 0xc00c47de18, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00508dc00, 0x7f06919ac748, 0xc000eff920, 0x5, 0xc000eff920, 0xc002db49c8) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00508dc00, 0x0, 0x0, 0xc0193a2c00) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00508dc00, 0xc0043a6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc000effda0, 0xc00309c8f8, 0x9, 0x9, 0xc00c47de00, 0xc002db4ce0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc000effda0, 0xc00309c8f8, 0x9, 0x9, 0x9, 0x1, 0xc00c47de00, 0x0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00309c8f8, 0x9, 0x9, 0x58a03a0, 0xc000effda0, 0x0, 0xc000000000, 0x100, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00309c8c0, 0xc0028039b0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00262fe00, 0xc002803b00, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0029bc1c0, 0x5981b58, 0xc00262fe00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0029bc1c0, 0x5981b58, 0xc00262fe00, 0xc000660ed0, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5300 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0001517c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 5472 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc005c64d80, 0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:466 +0x33e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc003b30fc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5635 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func1(0x59223f8, 0xc005eff920) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:754 +0xef | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff920, 0x0, 0x0, 0xc000de3bf0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5684 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00219f980) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5685 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046c88, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000b49418, 0x72, 0xeb00, 0xeb22, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000b49400, 0xc00a892000, 0xeb22, 0xeb22, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000b49400, 0xc00a892000, 0xeb22, 0xeb22, 0xc00a892005, 0xc000bd7418, 0xc004668828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc005abd388, 0xc00a892000, 0xeb22, 0xeb22, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc005fa1b60, 0xc00a892000, 0xeb22, 0xeb22, 0x428e5b, 0xc00fab8378, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc00fab8378, 0xc00a892000, 0xeb22, 0xeb22, 0xeb1d, 0xc000aaa000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc001a87e78, 0x58a0580, 0xc00fab8378, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc001a87c00, 0x7f06919ac748, 0xc005fa1b60, 0x5, 0xc005fa1b60, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc001a87c00, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc001a87c00, 0xc006598000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc006157860, 0xc000a8d1b8, 0x9, 0x9, 0xc004668d28, 0x420ee5, 0xc005cdbce0) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc006157860, 0xc000a8d1b8, 0x9, 0x9, 0x9, 0xc00fab8360, 0x481e60, 0xc005cdbce0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000a8d1b8, 0x9, 0x9, 0x58a03a0, 0xc006157860, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000a8d180, 0xc0288d0640, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00219f980, 0xc006166a20, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0029bc1c0, 0x5981b58, 0xc00219f980) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0029bc1c0, 0x5981b58, 0xc00219f980, 0xc00375b710, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5301 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc0012dab00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5302 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000aef440, 0xc000151770, 0xc0003ce820, 0xf, 0xc003fe5f20, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc00063f4c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc00063f4c0, 0x0, 0x0, 0xc0001518b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5441 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694047110, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001462c18, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001462c00, 0xc006158000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001462c00, 0xc006158000, 0x8b5, 0x8b5, 0x7f068d4cb0e8, 0x10, 0x155) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000c62660, 0xc006158000, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc01dbfcbe8, 0xc006158000, 0x8b5, 0x8b5, 0x5167a5, 0xc000aaa000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0021273f8, 0x58a0580, 0xc01dbfcbe8, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc002127180, 0x58a88a0, 0xc000c62660, 0x5, 0xc000c62660, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc002127180, 0x0, 0x0, 0xc027d521e0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc002127180, 0xc006580000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc006157200, 0xc000a8cff8, 0x9, 0x9, 0x7f068d596b00, 0x48, 0xc0284633d8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc006157200, 0xc000a8cff8, 0x9, 0x9, 0x9, 0xc027d30001, 0xc00633fd38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000a8cff8, 0x9, 0x9, 0x58a03a0, 0xc006157200, 0x0, 0xc000000000, 0xaba293, 0xc0038850e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000a8cfc0, 0xc027d359f0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0038850e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5682 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001309c70, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006157320, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc0038850e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 5275 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc003464f00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 5437 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc000d4edc0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5505 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x10) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b310e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5506 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0xe) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b310f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5507 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0xf) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31100) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5508 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x13) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31110) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5509 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x12) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31120) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5510 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x16) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5511 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x14) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31140) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5512 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x15) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31150) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5513 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x19) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31160) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5514 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x17) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31170) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5515 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x18) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31180) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5516 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x1b) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31190) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5517 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x1a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b311a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5518 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x1d) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b311b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5519 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x1c) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b311c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5520 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x1e) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b311d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5521 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x1f) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b311e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5522 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x20) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b311f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5523 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x21) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31200) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5524 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x24) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31210) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5525 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x28) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31220) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5526 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x2a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31230) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5527 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x2c) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5528 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x30) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31250) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5529 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x2e) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31260) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5530 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x2f) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31270) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5531 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x22) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31280) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5532 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x25) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31290) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5533 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x27) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b312a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5534 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x2d) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b312b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5535 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x23) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b312c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5536 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x29) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b312d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5537 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x26) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b312e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5538 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x2b) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b312f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5539 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x31) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31300) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5540 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x33) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31310) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5541 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x34) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5542 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x32) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31330) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5543 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x37) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31340) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5544 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x35) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31350) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5545 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x38) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31360) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5546 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x36) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31370) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5547 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x3b) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5548 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x39) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31390) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5549 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x3c) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b313a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5550 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x3a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31410) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5551 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x3e) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31420) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5552 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x3f) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31430) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5553 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc003c403d0, 0x3d) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc003c403c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc002629ae0, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:251 +0x8b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31440) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5554 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc002629b70) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc002629b68) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:212 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31450) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5555 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop(0xc00250e000, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:626 +0x232 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31460) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5556 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop(0xc00250e000, 0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:688 +0x10e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b31470) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5557 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func2(0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1563 +0xb3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc006012078) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5558 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc000e47720) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5559 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc000e47770) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5560 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1725 +0x291 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc000e477c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5755 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0014bd720, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006764120, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc0060d3500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5562 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startLeaseRenewer.func1(0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1783 +0x2b0 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b314d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5440 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0038850e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 5432 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca76da8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000b48f98, 0x72, 0x4800, 0x48a9, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000b48f80, 0xc005fd6000, 0x48a9, 0x48a9, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000b48f80, 0xc005fd6000, 0x48a9, 0x48a9, 0x48a4, 0xc005fd6000, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc005abd0c8, 0xc005fd6000, 0x48a9, 0x48a9, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc006c4ddb8, 0xc005fd6000, 0x48a9, 0x48a9, 0x48a4, 0xc000c81400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc001a86278, 0x58a0580, 0xc006c4ddb8, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc001a86000, 0x58a88a0, 0xc005abd0c8, 0x5, 0xc005abd0c8, 0x9b) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc001a86000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc001a86000, 0xc006116000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc006107680, 0xc000a8cab8, 0x9, 0x9, 0x4221c5, 0xc00627cb40, 0xc0045fbce8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc006107680, 0xc000a8cab8, 0x9, 0x9, 0x9, 0xc0045fbd00, 0x5318d9, 0xc003c409c8) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000a8cab8, 0x9, 0x9, 0x58a03a0, 0xc006107680, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000a8ca80, 0xc005cf50e0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc003884f00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5565 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1(0x59223f8, 0xc005eff1a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1892 +0x2b5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1a0, 0x0, 0x0, 0xc003b314e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5566 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1(0x59223f8, 0xc005eff440) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_rebalancer.go:190 +0x16d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff440, 0x0, 0x0, 0xc0060120c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5567 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:678 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc000164340) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5568 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossiping.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:643 +0x1df | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0060120d8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5569 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func1(0xc003a54960, 0xc005ecb560) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:216 +0x9c | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:212 +0x185 | |
| goroutine 5570 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider.func1(0xc002959ab0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1126 +0xda | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1121 +0x24e | |
| goroutine 5571 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.periodicallyPersistHLCUpperBound(0xc000e29310, 0xc005ea2cb0, 0xc003b31630, 0x49b2640, 0xc00142aa80, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1088 +0x150 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startPersistingHLCUpperBound.func2(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1164 +0x69 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc005eff770) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5572 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports.(*Reporter).Start.func2(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports/reporter.go:152 +0x1cd | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc006012120) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5573 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.startSampleEnvironment.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2561 +0x1c9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0021c35e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5574 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:751 +0x43d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0021c3730) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5713 [IO wait, 39 minutes]: | |
| internal/poll.runtime_pollWait(0x7f06bca76f78, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000b48b18, 0x72, 0x0, 0x0, 0x435cb7e) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc000b48b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc000b48b00, 0xc000211900, 0xc0024fa000, 0x203001) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc00118dcf8, 0x0, 0xc000540640, 0xc000540668) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc00118dcf8, 0x8000, 0xc003b13400, 0x7f0691913308, 0x28) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc000116978, 0x59223f8, 0xc0005cdad0, 0xc002513080, 0x5905230, 0xc00118dcf8, 0xc0050a8540, 0xc000540708, 0x11c3f07) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:135 +0x50 | |
| github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2330 +0xfc | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc006bf2900) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5588 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc003298780, 0x5978db0, 0xc003db4420, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:190 +0x6c5 | |
| github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x420a960, 0xc003298780, 0x5966670, 0xc001633ec0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:660 +0xad | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x420a960, 0xc003298780, 0x5966670, 0xc001633ec0, 0xc005f94210, 0x49a94f8, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:170 +0x5f7 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x420a960, 0xc003298780, 0x5966670, 0xc001633ec0, 0x40c4c00, 0xc003b4cac0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x420a960, 0xc003298780, 0x5966670, 0xc001633ec0, 0xc005f94210, 0xc003b4cac0, 0xc003b4cac0, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:186 +0x99 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x420a960, 0xc003298780, 0x5966670, 0xc001633ec0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x1, 0x420a960, 0xc003298780, 0x5966670, 0xc001633ec0, 0xc005f94210, 0xc003b4ca00, 0x40c4c00, 0xc003b4ca00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb7 | |
| google.golang.org/grpc.chainStreamServerInterceptors.func1(0x420a960, 0xc003298780, 0x5966670, 0xc001633ec0, 0xc005f94210, 0x49a94f8, 0x59223f8, 0xc005f888d0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1401 +0xbd | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0029bc1c0, 0x5981b58, 0xc00487fc80, 0xc005c58b40, 0xc0015de750, 0x764e240, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1549 +0x6d8 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0029bc1c0, 0x5981b58, 0xc00487fc80, 0xc005c58b40, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1622 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001f28da0, 0xc0029bc1c0, 0x5981b58, 0xc00487fc80, 0xc005c58b40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:941 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:939 +0x1fd | |
| goroutine 5578 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc001fdb180, 0xc005c658c0, 0x59223f8, 0xc0062f2450) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 5389 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00487fc80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5390 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06bca76e90, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001550298, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001550280, 0xc00343d000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001550280, 0xc00343d000, 0x773, 0x773, 0xc00343d005, 0xc003c7b2d8, 0xc004d51828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00469aa58, 0xc00343d000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc002b35440, 0xc00343d000, 0x773, 0x773, 0x428e5b, 0xc00f793cc8, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc00f793cc8, 0xc00343d000, 0x773, 0x773, 0x76e, 0xc000aaa000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc001fdb778, 0x58a0580, 0xc00f793cc8, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc001fdb500, 0x7f06919ac748, 0xc002b35440, 0x5, 0xc002b35440, 0xc6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc001fdb500, 0x0, 0x0, 0x4) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc001fdb500, 0xc0060ba000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0060b80c0, 0xc00342ac78, 0x9, 0x9, 0xc005f8bf20, 0xc004d51ce0, 0x4221c5) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0060b80c0, 0xc00342ac78, 0x9, 0x9, 0x9, 0x498ff4, 0xc0059a8a80, 0xc0021ce900) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00342ac78, 0x9, 0x9, 0x58a03a0, 0xc0060b80c0, 0x0, 0x0, 0xbd, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00342ac40, 0xc0059a89c0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00487fc80, 0xc0059a8b10, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0029bc1c0, 0x5981b58, 0xc00487fc80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0029bc1c0, 0x5981b58, 0xc00487fc80, 0xc00369bd30, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5579 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0013c0280, 0xc0062ca610, 0x5, 0x5, 0x8, 0x594ae18, 0x45663c) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0013c0280, 0xc0062ca610, 0x5, 0x5, 0xc002139500, 0xc0005be030, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0062f2510, 0xc0062ca610, 0x5, 0x5, 0xc0045fa978, 0xc0045fa940, 0xacfcdc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc0062f2510, 0xc0062ca610, 0x5, 0x5, 0x5, 0x1ac2110, 0xc0004994a0, 0xc0045fa9cf) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc005c65b00, 0xc0062ca610, 0x5, 0x5, 0x0, 0xc0033a8760, 0xc02360fe00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc0062ca600, 0x7fffffff, 0x2, 0x2, 0x9, 0xa, 0xc02f274cb0, 0xc0001202a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc0062ca600, 0xc005c65b00, 0x0, 0x0, 0x7fffffff, 0xc0045fac60, 0x59058c0, 0x7a02de8, 0xc0045fab40, 0x485260, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc0062ca600, 0x7f06bca6db20, 0xc00044af90, 0xc005c65b00, 0x0, 0x0, 0x41ae880, 0xc023642960, 0x7fffffff, 0xc0045fac60, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc002352a00, 0x41ae880, 0xc023642960, 0xc0045fac60, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:957 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc002352a00, 0x8, 0x8d) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:808 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc005c658c0, 0xc0045fade0, 0xc0045fadb0, 0x50, 0x48) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:662 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc005c658c0, 0x41ae880, 0xc023642960, 0x0, 0xc001db6310) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:807 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc003b31e90, 0x5922350, 0xc003c40200, 0xc002603520) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:636 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0xc005ecb8c0, 0x5978d58, 0xc003b31e90, 0xc004f7b680, 0x5922350, 0xc003c40200, 0xc002603520, 0xc0050d8691, 0xc00208c160) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:324 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x5922350, 0xc003c40200) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:339 +0x9c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc003c40200, 0x0, 0x0, 0xc003c40a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5589 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000ef1a90, 0xc005f94208, 0x5, 0x5, 0xc01dfb2612, 0xc01b3de000, 0x40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:177 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000ef1a90, 0xc005f94208, 0x5, 0x5, 0x0, 0xc003d64480, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:171 +0x211 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc005f887e0, 0xc005f94208, 0x5, 0x5, 0x7a02de8, 0xc008b597c8, 0xad11dc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc005f887e0, 0xc005f94208, 0x5, 0x5, 0x5, 0xf92fa2a280749d01, 0xf90000c0059a8ab0, 0x1) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc005c58b40, 0xc005f94208, 0x5, 0x5, 0xc0065b8930, 0x7a02d01, 0x3c55740) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc005f941f8, 0x7fffffff, 0x3e62b80, 0xc00098a978, 0x1, 0xc01dfee060, 0x5a3dad6bb22, 0xc000aaa000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc005f941f8, 0xc005c58b40, 0x0, 0x0, 0x7fffffff, 0xc008b59b00, 0x59058c0, 0x7a02de8, 0x40, 0x40, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc005f941f8, 0x7f06bca6db20, 0xc00044af90, 0xc005c58b40, 0x0, 0x0, 0x41ae740, 0xc01dfec0a0, 0x7fffffff, 0xc008b59b00, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc001633ec0, 0x41ae740, 0xc01dfec0a0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:1545 +0x156 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc003db4420, 0x0, 0x5f96180, 0x49a94e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:679 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc003298780, 0x5922350, 0xc003b4cb00, 0xc005abd128, 0xc003b4cb40, 0xc008b59f08, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:331 +0x387 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3(0x5922350, 0xc003b4cb00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:139 +0x9e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc003b4cb00, 0x0, 0x0, 0xc003b4cb80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5580 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc005efed50) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005efed50, 0x0, 0x0, 0xc000165600) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5581 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc005efedb0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005efedb0, 0x0, 0x0, 0xc000165640) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5582 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc005efee10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005efee10, 0x0, 0x0, 0xc000165660) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5583 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc005efee70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005efee70, 0x0, 0x0, 0xc000165680) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5584 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc005efef30) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005efef30, 0x0, 0x0, 0xc0001656a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5585 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc005efef90) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005efef90, 0x0, 0x0, 0xc0001656c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5602 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc005efeff0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005efeff0, 0x0, 0x0, 0xc0001656e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5603 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc005eff050) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff050, 0x0, 0x0, 0xc000165700) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5604 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2(0x59223f8, 0xc005eff0e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:823 +0x12c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff0e0, 0x0, 0x0, 0xc000165720) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5605 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).waitAndProcess(0xc0008a2ea0, 0x59223f8, 0xc005eff1d0, 0x1f0b7ee1, 0xed8ac0461, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:219 +0x19d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1(0x59223f8, 0xc005eff1d0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:286 +0x309 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc005eff1d0, 0x0, 0x0, 0xc0062fe060) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5618 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc002290000, 0xc000dfb630, 0xc00610c3f0, 0xf, 0xc0059d48a0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc0039efe00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc0039efe00, 0x0, 0x0, 0xc000dfb860) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5712 [select]: | |
| github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1(0x5922388, 0xc000074130) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:166 +0xea | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074130, 0x0, 0x0, 0xc006b35350) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5590 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00139efa0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 5591 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc005ac5080) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5592 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc002290000, 0xc00139ef50, 0xc00610c690, 0xf, 0xc005f96420, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc0039efe00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc0039efe00, 0x0, 0x0, 0xc00139f0e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5702 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046ba0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000e94518, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000e94500, 0xc00341db00, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000e94500, 0xc00341db00, 0x8b5, 0x8b5, 0x8b0, 0xc00341db00, 0xc00643d7c8) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc002a662d0, 0xc00341db00, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc02a901050, 0xc00341db00, 0x8b5, 0x8b5, 0x7f0691942ee0, 0xc000280400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc001c4a5f8, 0x58a0580, 0xc02a901050, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc001c4a380, 0x58a88a0, 0xc002a662d0, 0x5, 0xc002a662d0, 0xc00643d960) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc001c4a380, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc001c4a380, 0xc0064c8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0064c6000, 0xc002dec038, 0x9, 0x9, 0x0, 0xc00643dc98, 0x58e6b70) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0064c6000, 0xc002dec038, 0x9, 0x9, 0x9, 0xc0000f7558, 0x9e7e783d60483a01, 0xc01eaeea20) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc002dec038, 0x9, 0x9, 0x58a03a0, 0xc0064c6000, 0x0, 0x0, 0xc01eacd388, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc002dec000, 0xc01eaee9c0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc002c3d0e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5622 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694047028, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001353998, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001353980, 0xc003f24900, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001353980, 0xc003f24900, 0x8b5, 0x8b5, 0x8b0, 0xc003f24900, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc0005cfc08, 0xc003f24900, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc01324c6c0, 0xc003f24900, 0x8b5, 0x8b5, 0x5167a5, 0xc002310400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00332a5f8, 0x58a0580, 0xc01324c6c0, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00332a380, 0x58a88a0, 0xc0005cfc08, 0x5, 0xc0005cfc08, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00332a380, 0x0, 0x0, 0xc028904870) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00332a380, 0xc006516000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc006514240, 0xc00342af18, 0x9, 0x9, 0x7f0688185c50, 0x48, 0xc0284635b8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc006514240, 0xc00342af18, 0x9, 0x9, 0x9, 0xc028900001, 0xc0044d2d38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00342af18, 0x9, 0x9, 0x58a03a0, 0xc006514240, 0x0, 0xc000000000, 0xaba293, 0xc00151fc20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00342aee0, 0xc02846f620, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00151fc20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5623 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00140a2d0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006514300, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00151fc20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 5628 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00140ad20, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006359020, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc0060d3380) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5629 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0060d3380) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5756 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0060d3500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5757 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046548, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0012dd018, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0012dd000, 0xc00675c000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0012dd000, 0xc00675c000, 0x773, 0x773, 0xc00675c005, 0xc000312dd8, 0xc0063ad828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc005abd900, 0xc00675c000, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0068ec240, 0xc00675c000, 0x773, 0x773, 0x428e5b, 0xc01ac05290, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc01ac05290, 0xc00675c000, 0x773, 0x773, 0x76e, 0xc000aaa000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0068ea5f8, 0x58a0580, 0xc01ac05290, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0068ea380, 0x7f06919ac748, 0xc0068ec240, 0x5, 0xc0068ec240, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0068ea380, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0068ea380, 0xc006766000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0067640c0, 0xc00342b458, 0x9, 0x9, 0xc0063add28, 0x420ee5, 0xc00673a5a0) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0067640c0, 0xc00342b458, 0x9, 0x9, 0x9, 0xc01ac05278, 0x481e60, 0xc00673a5a0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00342b458, 0x9, 0x9, 0x58a03a0, 0xc0067640c0, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00342b420, 0xc027868320, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0060d3500, 0xc00675a480, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc001f46fc0, 0x5981b58, 0xc0060d3500) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc001f46fc0, 0x5981b58, 0xc0060d3500, 0xc0063537e0, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5954 [select, 39 minutes]: | |
| database/sql.(*DB).connectionOpener(0xc000d6ec30, 0x5922350, 0xc006bf2a40) | |
| /usr/local/go/src/database/sql/sql.go:1133 +0xb5 | |
| created by database/sql.OpenDB | |
| /usr/local/go/src/database/sql/sql.go:740 +0x12a | |
| goroutine 5955 [sync.Cond.Wait, 39 minutes]: | |
| sync.runtime_notifyListWait(0xc006bf2e90, 0x1d05820000000013) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc006bf2e80) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc002edc638, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:441 +0x9e | |
| github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00224d000, 0x5922350, 0xc006bf2ec0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1588 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00224d000, 0x5922350, 0xc006bf2ec0, 0xc000a3f9a0, 0x5400, 0x15000, 0xc000a3fc20, 0xc006b359b0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1564 +0x225 | |
| github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc000ed2c00, 0x5922350, 0xc006bf2ec0, 0xc00224d000, 0x5400, 0x15000, 0xc000a3fc20, 0xc006b359b0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:648 +0xce | |
| github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc00040b9b0, 0xc006b53f68, 0x5922350, 0xc006bf2ec0, 0xc006b359b0, 0xc000ed2c00, 0xc002edc000, 0x597cae0, 0xc003b13d80, 0xc00150f3e0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:645 +0x3f5 | |
| created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:573 +0x185 | |
| goroutine 5596 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046d70, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000e94298, 0x72, 0xd800, 0xd840, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000e94280, 0xc006e44000, 0xd840, 0xd840, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000e94280, 0xc006e44000, 0xd840, 0xd840, 0xd83b, 0xc006e44000, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc002a66180, 0xc006e44000, 0xd840, 0xd840, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc01324c660, 0xc006e44000, 0xd840, 0xd840, 0x5167a5, 0xc002310400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc001fdbaf8, 0x58a0580, 0xc01324c660, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc001fdb880, 0x58a88a0, 0xc002a66180, 0x5, 0xc002a66180, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc001fdb880, 0x0, 0x0, 0xc028904780) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc001fdb880, 0xc00636e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc006358d80, 0xc001347c38, 0x9, 0x9, 0x7f0688185c50, 0x48, 0xc0284631f8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc006358d80, 0xc001347c38, 0x9, 0x9, 0x9, 0xc028900001, 0xc0063aad38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc001347c38, 0x9, 0x9, 0x58a03a0, 0xc006358d80, 0x0, 0xc000000000, 0xaba293, 0xc003972000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc001347c00, 0xc02846f610, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc003972000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5597 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139f9a0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006358e40, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc003972000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 5703 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001446910, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0064c60c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc002c3d0e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 5754 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0014bd180, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00657fbc0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00673e5a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 5753 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046718, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0012dcf18, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0012dcf00, 0xc002ca2400, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0012dcf00, 0xc002ca2400, 0x8b5, 0x8b5, 0x8b0, 0xc002ca2400, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc005abd8d8, 0xc002ca2400, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc01ac05248, 0xc002ca2400, 0x8b5, 0x8b5, 0x5167a5, 0xc000aaa000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0065b9e78, 0x58a0580, 0xc01ac05248, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0065b9c00, 0x58a88a0, 0xc005abd8d8, 0x5, 0xc005abd8d8, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0065b9c00, 0x0, 0x0, 0xc027861c50) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0065b9c00, 0xc006742000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc00657fb00, 0xc00342b298, 0x9, 0x9, 0x7f069049d098, 0x48, 0xc027849df8) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc00657fb00, 0xc00342b298, 0x9, 0x9, 0x9, 0xc027800001, 0xc0065e1d38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00342b298, 0x9, 0x9, 0x58a03a0, 0xc00657fb00, 0x0, 0xc000000000, 0xaba293, 0xc00673e5a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00342b260, 0xc027868310, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00673e5a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5636 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache.(*Cache).periodicallyRefreshProtectedtsCache(0xc0033a88f0, 0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache/cache.go:160 +0x239 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc002dda490) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5637 [select, 4 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).run(0xc0011e2cc0, 0x59223f8, 0xc0005cdad0, 0xc002513080) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:121 +0x296 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).Start.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:100 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc000de3c08) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5638 [select, 10 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startSystemLogsGC.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server_systemlog_gc.go:177 +0x545 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc000de3c20) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5736 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc001467590) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 5737 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc005ac5600) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5738 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc002290240, 0xc001467540, 0xc00369bfa0, 0xf, 0xc005f97800, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc001f02580) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc001f02580, 0x0, 0x0, 0xc001467680) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5691 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0038852c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 5611 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046800, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0012dce18, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0012dce00, 0xc002ca2d00, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0012dce00, 0xc002ca2d00, 0x8b5, 0x8b5, 0x8b0, 0xc002ca2d00, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc005abd8e8, 0xc002ca2d00, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc00b243998, 0xc002ca2d00, 0x8b5, 0x8b5, 0x5167a5, 0xc000382400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0068ea278, 0x58a0580, 0xc00b243998, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0068ea000, 0x58a88a0, 0xc005abd8e8, 0x5, 0xc005abd8e8, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0068ea000, 0x0, 0x0, 0xc029cb13b0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0068ea000, 0xc0063ea000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0063e2540, 0xc0001a6f18, 0x9, 0x9, 0x7f068d566d30, 0x48, 0xc02916e778) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0063e2540, 0xc0001a6f18, 0x9, 0x9, 0x9, 0xc029cc0001, 0xc0063dfd38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0001a6f18, 0x9, 0x9, 0x58a03a0, 0xc0063e2540, 0x0, 0xc000000000, 0xaba293, 0xc0034654a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001a6ee0, 0xc029cbe200, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0034654a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 7343 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0027d47e0, 0x3, 0xc006cf7aa0, 0xc003522380, 0x5978eb8, 0xc00556c800, 0x5978e01, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:481 +0x56d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:632 +0x366 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc005eff950) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5944 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046378, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000f22f98, 0x72, 0x700, 0x731, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000f22f80, 0xc0018a5000, 0x731, 0x731, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000f22f80, 0xc0018a5000, 0x731, 0x731, 0x0, 0x58a8640, 0x7a02600) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000c62790, 0xc0018a5000, 0x731, 0x731, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc02313cab0, 0xc0018a5000, 0x731, 0x731, 0x76c4a60, 0xc000604000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0068ebaf8, 0x58a0580, 0xc02313cab0, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0068eb880, 0x58a88a0, 0xc000c62790, 0x5, 0xc000c62790, 0x3e71220) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0068eb880, 0x0, 0x0, 0x856a11) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0068eb880, 0xc000ef8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc006b53f50, 0xc000ef8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1736 +0x12a | |
| bufio.(*Reader).fill(0xc002edc060) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x108 | |
| bufio.(*Reader).ReadByte(0xc002edc060, 0x2b0c5913, 0xed8abfb04, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:253 +0x39 | |
| github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc002edc830, 0x58ef090, 0xc002edc060, 0x58e8380, 0x7a02de8, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:203 +0x35 | |
| github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func4(0xc002edc000, 0xc006b35980, 0xc0055497b3, 0xc0055497b4, 0x58fd0a0, 0xc003b13d80, 0xc005549838, 0xc000ed2c00, 0xc0055497b2, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:339 +0x71 | |
| github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc002edc000, 0x59223f8, 0xc004dfc780, 0xc006b35970, 0xc000ed2c00, 0x5400, 0x15000, 0xc000a3fc20, 0x0, 0x4, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:470 +0x5d7 | |
| github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc003327d00, 0x59223f8, 0xc004dfc780, 0x5978bf8, 0xc0068eb880, 0xc00376301e, 0x4, 0xc004dfc750, 0x58e7688, 0xc0050a85a0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:160 +0x1d8 | |
| github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc003327d00, 0x59223f8, 0xc004dfc780, 0x597dab0, 0xc0068eb880, 0x5922301, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:692 +0xe25 | |
| github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x597dab0, 0xc000c62790) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2334 +0xf7 | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0xc000c62680, 0x597dab0, 0xc000c62790, 0xc003e24630) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf | |
| created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x256 | |
| goroutine 5640 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage.(*Storage).deleteSessionsLoop(0xc0033a8f20, 0x59223f8, 0xc00396bdc0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage/slstorage.go:317 +0x1fa | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0067a2490) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5641 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoop(0xc0011e32c0, 0x5922350, 0xc00396be00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:214 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0067a24f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5642 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier.(*Notifier).run(0xc001e2ba40, 0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier/notifier.go:136 +0xea | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0067a2500) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5643 [select, 10 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*TemporaryObjectCleaner).Start.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/temporary_schema.go:597 +0x2de | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc000de3dd0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5644 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowScheduler).Start.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow_scheduler.go:263 +0x13d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0067a2540) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5645 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/idxusage.(*LocalIndexUsageStats).startStatsIngestionLoop.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/idxusage/local_idx_usage_stats.go:408 +0x10b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc000de3de8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5646 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0033bb2a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5647 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0033bb2c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5648 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).startSQLStatsFlushLoop.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/provider.go:98 +0x1c5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc000de3e00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5649 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).periodicallyClearSQLStats.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:89 +0x315 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0033bb2e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5778 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/stats/automatic_stats.go:260 +0x18a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0033bb300) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5779 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics.(*Registry).poll(0xc0008f4900, 0x5922350, 0xc00396b0c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics/statement_diagnostics.go:147 +0x2ba | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc00396b0c0, 0x0, 0x0, 0xc0067a2580) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5780 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc002561ba0, 0x5922350, 0xc00396b140, 0xc0067b2480, 0xc0067f41e0, 0xc0067f4240, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xe7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc002561ba0, 0x5922350, 0xc00396b140, 0xc0067b2480) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:226 +0x5f8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*Factory).RangeFeed.func1(0x5922350, 0xc00396b140) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:137 +0x47 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc00396b140, 0x0, 0x0, 0xc000de3e90) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5781 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).RefreshLeases.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:898 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0067b2540) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5782 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).PeriodicallyRefreshSomeLeases.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1007 +0x2ec | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0067a25e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5783 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func7(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:817 +0x1ef | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0033bb460) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5784 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func8(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:849 +0x1d9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc000de3ec0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5785 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func9(0x5922388, 0xc0067e2480) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/registry.go:872 +0x215 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc0067b2600) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7254 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001376d70, 0xc003760520, 0x5, 0x5, 0xc007545978, 0x12, 0x33) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:177 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001376d70, 0xc003760520, 0x5, 0x5, 0xc00583fc00, 0xc002bf48d0, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:171 +0x211 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc005db16b0, 0xc003760520, 0x5, 0x5, 0x1c21585, 0xc007545a40, 0xad11dc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc005db16b0, 0xc003760520, 0x5, 0x5, 0x5, 0xc007545ab0, 0x426f05, 0x3da3b20) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0022c2900, 0xc003760520, 0x5, 0x5, 0x41e270, 0xc007545c68, 0xc0005b5f80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc003760510, 0x7fffffff, 0xc007545be8, 0xc0065b8998, 0xee08eb1af70d2601, 0xee0000c0024c7320, 0x2, 0xc0065b8970) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc003760510, 0xc0022c2900, 0x0, 0x0, 0x7fffffff, 0xc007545d78, 0x59058c0, 0x7a02de8, 0x45663c, 0xc007545c48, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc003760510, 0x7f06bca6db20, 0xc00044af90, 0xc0022c2900, 0x0, 0x0, 0x41b3100, 0xc00920ef00, 0x7fffffff, 0xc007545d78, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc00340e0c0, 0x41b3100, 0xc00920ef00, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:1545 +0x156 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc0050b6c40, 0x593db70, 0xc001ccced0, 0xc00583fc00) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:187 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:83 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1(0x5978f10, 0xc0050b6c40, 0xc0027d47e0, 0x593db70, 0xc001ccced0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:340 +0x82 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1(0x593db70, 0xc001ccced0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:376 +0x5d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x593db70, 0xc001ccced0, 0xc001cccd20, 0x0, 0xc005db18f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5727 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc002e2cc80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 5885 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc006d1f860) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 7280 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc001390000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5795 [semacquire, 39 minutes]: | |
| sync.runtime_Semacquire(0xc0067e6d90) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc0067e6d88) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0067e6d80, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait(0xc0067e6d80, 0x5922350, 0xc0067e2400, 0xc0067e4e40, 0xc0067e2400) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:142 +0x57 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc000ebc000, 0x5922350, 0xc0067e6d20, 0xc0067aae70, 0x1, 0x8, 0xc0067aae78, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:83 +0x3ca | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc000e6ccf0, 0x5922350, 0xc00396b140, 0xc0067aae70, 0x1, 0x8, 0xc0067aae78, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:69 +0xb4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1(0x5922350, 0xc00396b140) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:220 +0xe3 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x5922350, 0xc00396b140, 0x0, 0x0, 0xc0067e6cf0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5796 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1(0x5922350, 0xc0067e2400, 0xc002513080, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:66 +0x12f | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0067e6d80, 0xc0067e6db0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 5612 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0013c1950, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0063e2600, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc0034654a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 5886 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0694046290, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0038c3298, 0x72, 0x4900, 0x4931, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0038c3280, 0xc00365aa80, 0x4931, 0x4931, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0038c3280, 0xc00365aa80, 0x4931, 0x4931, 0x492c, 0xc00365aa80, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc002c9a370, 0xc00365aa80, 0x4931, 0x4931, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc022e12780, 0xc00365aa80, 0x4931, 0x4931, 0x492c, 0xc000604000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc003c025f8, 0x58a0580, 0xc022e12780, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc003c02380, 0x58a88a0, 0xc002c9a370, 0x5, 0xc002c9a370, 0xc0065e8960) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc003c02380, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc003c02380, 0xc007040000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc0048845a0, 0xc002f67c38, 0x9, 0x9, 0x0, 0xc0065e8c98, 0x58e6b70) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc0048845a0, 0xc002f67c38, 0x9, 0x9, 0x9, 0xc0000f7558, 0x9e7e783d60483a01, 0xc029bb63c0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc002f67c38, 0x9, 0x9, 0x58a03a0, 0xc0048845a0, 0x0, 0x0, 0xc0172478d8, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc002f67c00, 0xc029bb6360, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc006d1f860) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5729 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc002290240, 0xc002e2cc30, 0xc0034f5390, 0xf, 0xc00042a360, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc001f02580) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc001f02580, 0x0, 0x0, 0xc002e2cd70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5802 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001d586e0, 0xc0067cb3a8, 0x5, 0x5, 0xffc03c, 0xc006b8cdf0, 0xc012bb7a0c) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001d586e0, 0xc0067cb3a8, 0x5, 0x5, 0xc00348f2c0, 0xc0067b6468, 0x49b2398) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0067e7440, 0xc0067cb3a8, 0x5, 0x5, 0x0, 0xc00348f320, 0xacfcdc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x58a8020, 0xc0067e7440, 0xc0067cb3a8, 0x5, 0x5, 0x5, 0xc00348f390, 0x426f05, 0x3da3b20) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0067f2fc0, 0xc0067cb3a8, 0x5, 0x5, 0xc012bb7a00, 0x16, 0x200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc0067cb398, 0x7fffffff, 0x19e2de2, 0x7a02de8, 0xc012bb7a00, 0x8, 0x200, 0x4252220) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:559 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc0067cb398, 0xc0067f2fc0, 0x0, 0x0, 0x7fffffff, 0xc00348f640, 0x59058c0, 0x7a02de8, 0xc0233344e0, 0x5a3df06fdd7, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:690 +0x4d | |
| google.golang.org/grpc.recv(0xc0067cb398, 0x7f06bca6db20, 0xc00044af90, 0xc0067f2fc0, 0x0, 0x0, 0x4252220, 0xc00fc9a780, 0x7fffffff, 0xc00348f640, ...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:758 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc000a93a00, 0x4252220, 0xc00fc9a780, 0xc00348f640, 0x45663c, 0xc00348f6a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:957 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000a93a00, 0xc00348f6e8, 0x421105) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:808 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc0067f2d80, 0xc00348f7c0, 0xc00348f790, 0x18, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:662 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc0067f2d80, 0x4252220, 0xc00fc9a780, 0x2, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:807 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/roachpb.(*internalRangeFeedClient).Recv(0xc0067a30b0, 0xc00348f8c8, 0x0, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8375 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc000ebc000, 0x5922350, 0xc0067e2400, 0xc0067aae70, 0x1, 0x8, 0xc0067aae78, 0x1, 0x1, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:270 +0x911 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc000ebc000, 0x5922350, 0xc0067e2400, 0xc0067b1490, 0x0, 0xc0067e4de0, 0xc0067f41e0, 0x1150273, 0x5922388) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:151 +0x34c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1(0x5922350, 0xc0067e2400, 0xc002513080, 0x49aeae8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:70 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1(0x0, 0x49aeb88) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:166 +0x3a | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0067e6d80, 0xc0067e7260) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 5803 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc001a87880, 0xc0067f2d80, 0x59223f8, 0xc0067e7380) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 7338 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002fd2780, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc005c07b60, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc003b4ed20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 5833 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06940468e8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0012dc898, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0012dc880, 0xc0033d4800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0012dc880, 0xc0033d4800, 0x773, 0x773, 0xc0033d4805, 0xc000c8a8d8, 0xc0067f8828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc005abd610, 0xc0033d4800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0068b4f00, 0xc0033d4800, 0x773, 0x773, 0x428e5b, 0xc00ddf7ed8, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc00ddf7ed8, 0xc0033d4800, 0x773, 0x773, 0x76e, 0xc000aaa000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0065b93f8, 0x58a0580, 0xc00ddf7ed8, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0065b9180, 0x7f06919ac748, 0xc0068b4f00, 0x5, 0xc0068b4f00, 0xd) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0065b9180, 0x0, 0x0, 0xa9cdbe) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0065b9180, 0xc006a22000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc006a18600, 0xc000171c38, 0x9, 0x9, 0xc0067f8d28, 0x420ee5, 0xc0067f52c0) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc006a18600, 0xc000171c38, 0x9, 0x9, 0x9, 0xc00ddf7ec0, 0x481e60, 0xc0067f52c0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000171c38, 0x9, 0x9, 0x58a03a0, 0xc006a18600, 0x0, 0xc000000000, 0x429501, 0x100000000000018) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000171c00, 0xc0291e7e90, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0067fec00, 0xc006a0dec0, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc001f46fc0, 0x5981b58, 0xc0067fec00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc001f46fc0, 0x5981b58, 0xc0067fec00, 0xc006352f20, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5775 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc001500870) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 5776 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc005ac58c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5777 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc0036ac900, 0xc001500820, 0xc005bee120, 0xf, 0xc0068b08a0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc0026d5240) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922350, 0xc0026d5240, 0x0, 0x0, 0xc001500960) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5809 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc00013a000, 0x6, 0xc0054e8f00, 0x596be38, 0xc0067a3580, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:257 +0x670 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:143 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc00250e800, 0xc0054e8f00, 0x596be38, 0xc0067a3580, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2530 +0x136 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc001e268c0, 0xc0054e8f00, 0x596be38, 0xc0067a3580, 0x10) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:216 +0xce | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc00283cc00, 0xc0054e8f00, 0x596be38, 0xc0067a3580, 0xc00283cc00, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:1101 +0x54 | |
| github.com/cockroachdb/cockroach/pkg/roachpb._Internal_RangeFeed_Handler(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0026480c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:8526 +0x113 | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0026480c0, 0xc0067cb920, 0x49aa000, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:170 +0x5f7 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0026480c0, 0x40c4c00, 0xc0067e2cc0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0026480c0, 0xc0067cb920, 0xc0067e2cc0, 0xc0067e2cc0, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:186 +0x99 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0026480c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x1, 0x42ead40, 0xc00283cc00, 0x5966670, 0xc0026480c0, 0xc0067cb920, 0xc0067e2c80, 0x40c4c00, 0xc0067e2c80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb7 | |
| google.golang.org/grpc.chainStreamServerInterceptors.func1(0x42ead40, 0xc00283cc00, 0x5966670, 0xc0026480c0, 0xc0067cb920, 0x49aa000, 0x59223f8, 0xc0067e7ec0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1401 +0xbd | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0029bc1c0, 0x5981b58, 0xc00219f980, 0xc0067f3440, 0xc0015df830, 0x765a660, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1549 +0x6d8 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0029bc1c0, 0x5981b58, 0xc00219f980, 0xc0067f3440, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1622 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00610ddf0, 0xc0029bc1c0, 0x5981b58, 0xc00219f980, 0xc0067f3440) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:941 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:939 +0x1fd | |
| goroutine 5728 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc002246000) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 7310 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc006c1d0c0, 0x593db70, 0xc006bbe570) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:357 +0x15f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc00256de80, 0x593db70, 0xc006bbe570, 0x43d650a, 0x29, 0xc003637fb0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:307 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:312 +0x17e | |
| created by time.goFunc | |
| /usr/local/go/src/time/sleep.go:180 +0x45 | |
| goroutine 5816 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0015013b0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 5817 [chan receive, 39 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc005ac5b80) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1219 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:849 +0x12a | |
| goroutine 5818 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000aef440, 0xc001501360, 0xc0060030a0, 0xf, 0xc0068b1320, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1163 +0x2a8 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1(0x5922350, 0xc00063f4c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1091 +0x7c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922350, 0xc00063f4c0, 0x0, 0x0, 0xc0015014f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5826 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc000171340, 0x5922350, 0xc0067e2e00, 0xc0067e2e00, 0xc0067a35a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc000171340, 0x5922350, 0xc0067e2e00, 0x6) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:276 +0xd4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x5922350, 0xc000a76ac0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:263 +0x6d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc000a76ac0, 0x0, 0x0, 0xc0067cb998) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5831 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00183ad20, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006a18660, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc0067fec00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 5832 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0067fec00) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5610 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0034654a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 5752 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00673e5a0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1527 +0x17a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:341 +0x178b | |
| goroutine 5692 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06940469d0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0012dc698, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0012dc680, 0xc006158900, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0012dc680, 0xc006158900, 0x8b5, 0x8b5, 0x8b0, 0xc006158900, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc005abd600, 0xc006158900, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc00920f710, 0xc006158900, 0x8b5, 0x8b5, 0x5167a5, 0xc000280400, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc0065b9078, 0x58a0580, 0xc00920f710, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc0065b8e00, 0x58a88a0, 0xc005abd600, 0x5, 0xc005abd600, 0x435a2c6) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0065b8e00, 0x0, 0x0, 0xc0281592c0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc0065b8e00, 0xc00691a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc00690e660, 0xc000a8d6f8, 0x9, 0x9, 0x7f068e5f3d60, 0x48, 0xc0291d9f78) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc00690e660, 0xc000a8d6f8, 0x9, 0x9, 0x9, 0xc028110001, 0xc006685d38, 0xab8b6d) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000a8d6f8, 0x9, 0x9, 0x58a03a0, 0xc00690e660, 0x0, 0xc000000000, 0xaba293, 0xc0038852c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000a8d6c0, 0xc0281532e0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0038852c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 5693 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001435680, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00690e780, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc0038852c0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae | |
| goroutine 7253 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0027d47e0, 0x5978f10, 0xc0050b6c40, 0x58e4870, 0xc0027d47e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:381 +0x210 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler(0x4281ba0, 0xc0027d47e0, 0x5965e90, 0xc006d858a0, 0x79a46f0, 0xc007545888) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:168 +0xad | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1(0x4281ba0, 0xc0027d47e0, 0x5966670, 0xc00340e0c0, 0xc0037605a0, 0x49a99e8, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:185 +0x45b | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x4281ba0, 0xc0027d47e0, 0x5966670, 0xc00340e0c0, 0x40c4c00, 0xc0050d4900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2(0x4281ba0, 0xc0027d47e0, 0x5966670, 0xc00340e0c0, 0xc0037605a0, 0xc0050d4900, 0xc0050d4900, 0x2) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:186 +0x99 | |
| google.golang.org/grpc.getChainStreamHandler.func1(0x4281ba0, 0xc0027d47e0, 0x5966670, 0xc00340e0c0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1415 +0xdd | |
| github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor(0x1, 0x4281ba0, 0xc0027d47e0, 0x5966670, 0xc00340e0c0, 0xc0037605a0, 0xc0050d48c0, 0x40c4c00, 0xc0050d48c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:97 +0xb7 | |
| google.golang.org/grpc.chainStreamServerInterceptors.func1(0x4281ba0, 0xc0027d47e0, 0x5966670, 0xc00340e0c0, 0xc0037605a0, 0x49a99e8, 0x59223f8, 0xc005db17d0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1401 +0xbd | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0029bc1c0, 0x5981b58, 0xc0054e1c80, 0xc0022c2900, 0xc0015df350, 0x765a620, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1549 +0x6d8 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0029bc1c0, 0x5981b58, 0xc0054e1c80, 0xc0022c2900, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1622 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0006fb8c0, 0xc0029bc1c0, 0x5981b58, 0xc0054e1c80, 0xc0022c2900) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:941 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:939 +0x1fd | |
| goroutine 5695 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*jobScheduler).runDaemon.func1(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/jobs/job_scheduler.go:357 +0x272 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0061652d8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5696 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Sender).Run.func2(0x59223f8, 0xc0005cdad0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:253 +0x1a9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc0026c50c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 5858 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1(0x59223f8, 0xc0063f2740) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:151 +0x2ca | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc002513080, 0x59223f8, 0xc0005cdad0, 0x0, 0x0, 0xc006165320) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 741071 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000ce57a0, 0x1, 0xc017b9a1e0, 0xc0005f1cc0, 0x5978eb8, 0xc0234f8480, 0x5978e01, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:481 +0x56d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:632 +0x366 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc01c49c0c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 819973 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1.1(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/stats/automatic_stats.go:272 +0xf9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc02a238a80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7093 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00509cb60, 0x5922350, 0xc0071f8b40, 0xc0071f8b40, 0xc0049ab330) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00509cb60, 0x5922350, 0xc0071f8b40, 0x2b) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:276 +0xd4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x5922350, 0xc0071f8a40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:263 +0x6d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc0071f8a40, 0x0, 0x0, 0xc00607ad80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7091 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc0071f09a0, 0x59223f8, 0xc004d66120, 0x2b, 0xc00607ad08, 0xc000e3e380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:242 +0x40c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1(0x59223f8, 0xc004d66120) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:197 +0x59 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc004d66120, 0x0, 0x0, 0xc004d401a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7090 [chan receive, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc0053a9500, 0x2b, 0xc0024c4c30, 0x5970988, 0xc004d40120, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:257 +0x670 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_rangefeed.go:143 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc00250e800, 0xc0024c4c30, 0x5970988, 0xc004d40120, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2530 +0x136 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc001e268c0, 0xc0024c4c30, 0x5970988, 0xc004d40120, 0xc002030768) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:216 +0xce | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc00283cc00, 0xc0024c4c30, 0x5970988, 0xc004d40120, 0xc002030768, 0x199c23a) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:1101 +0x54 | |
| github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func1(0xc0049ab2e0, 0x596bed8, 0xc00283cc00, 0xc0024c4c30, 0x5922350, 0xc0071f8a00, 0xc004d3e180, 0xc004d3e1e0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:590 +0xc5 | |
| created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed | |
| /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:588 +0x12e | |
| goroutine 7089 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00509ca80, 0x5922350, 0xc0071f8700, 0xc0071f8700, 0xc0049aab80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00509ca80, 0x5922350, 0xc0071f8700, 0x2c) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:276 +0xd4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x5922350, 0xc0071f85c0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:263 +0x6d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc0071f85c0, 0x0, 0x0, 0xc00607a7c8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7087 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc0071f0840, 0x59223f8, 0xc0049a92f0, 0x2c, 0xc00607a738, 0xc000e3e380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:242 +0x40c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1(0x59223f8, 0xc0049a92f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:197 +0x59 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0049a92f0, 0x0, 0x0, 0xc0049c3840) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 6617 [select, 39 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc.(*Allocator).start.func1(0x59223f8, 0xc0014fd380) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc/id_alloc.go:151 +0x3cd | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc0014fd380, 0x0, 0x0, 0xc0051e6060) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7037 [select, 39 minutes]: | |
| google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00227e240) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:322 +0x99 | |
| google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:337 | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc001937a80, 0x41b3240, 0xc004bff140, 0xc003b37c58, 0xc005761f00, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:943 +0x731 | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001937a80, 0x8, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:808 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00227e000, 0xc005850dd8, 0xc005850da8, 0x40, 0x40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:662 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00227e000, 0x41b3240, 0xc004bff140, 0x79b3520, 0xc005761fc0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:807 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc0044ca3a0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:109 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1(0x5978eb8, 0xc0044ca3a0, 0xc0055cdcc0, 0xc0027d47e0, 0x59223f8, 0xc00548ba70, 0x0, 0xc005815420) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:456 +0x3b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1(0x59223f8, 0xc00548ba70) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:471 +0x66 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc00548ba70, 0x0, 0x0, 0xc00548bb30) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7330 [select, 39 minutes]: | |
| google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00227f0e0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:322 +0x99 | |
| google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:337 | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc005d74b80, 0x41b3240, 0xc004bff9c0, 0xc0067f6c58, 0x436a4e6, 0xc) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:943 +0x731 | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc005d74b80, 0x8, 0xc006c96b40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:808 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00227eea0, 0xc00484a5d8, 0xc00484a5a8, 0x40, 0x40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:662 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00227eea0, 0x41b3240, 0xc004bff9c0, 0xc001694847, 0x375c27ffa54) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:807 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc0044cafc0, 0x79b3520, 0x1694847, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:109 +0x62 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1(0x5978eb8, 0xc0044cafc0, 0xc0005f1bc0, 0xc000ce57a0, 0x59223f8, 0xc006a45410, 0xc0033b6780, 0x5922388) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:456 +0x3b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1(0x59223f8, 0xc006a45410) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:471 +0x66 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc006a45410, 0x0, 0x0, 0xc006a45500) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7344 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc007093500, 0xc0061e0c60, 0x59223f8, 0xc005effa40) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 7319 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000ce57a0, 0x1, 0xc004cba780, 0xc0005f1bc0, 0x5978eb8, 0xc0044cafc0, 0x5978e00, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:481 +0x56d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2(0x59223f8, 0xc001ca6ea0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:632 +0x366 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x59223f8, 0xc001ca6ea0, 0x0, 0x0, 0xc006a87cb0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7339 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc007093500, 0xc00532cea0, 0x59223f8, 0xc005c11ce0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 7337 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06905c3580, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0051dc898, 0x72, 0x800, 0x8b5, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0051dc880, 0xc002ca0900, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0051dc880, 0xc002ca0900, 0x8b5, 0x8b5, 0x8b0, 0xc002ca0900, 0x5) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc004e63338, 0xc002ca0900, 0x8b5, 0x8b5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| crypto/tls.(*atLeastReader).Read(0xc029d22ab0, 0xc002ca0900, 0x8b5, 0x8b5, 0x8b0, 0x79b7300, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc007093af8, 0x58a0580, 0xc029d22ab0, 0x426be5, 0x3da8720, 0x42944e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc007093880, 0x58a88a0, 0xc004e63338, 0x5, 0xc004e63338, 0xc007540960) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc007093880, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc007093880, 0xc006e80000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc005c07aa0, 0xc00509c818, 0x9, 0x9, 0x0, 0xc007540c98, 0x58e6b70) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc005c07aa0, 0xc00509c818, 0x9, 0x9, 0x9, 0xc0000f7558, 0x9e7e783d60483a01, 0xc00cfb9980) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00509c818, 0x9, 0x9, 0x58a03a0, 0xc005c07aa0, 0x0, 0x0, 0xc010c0bc48, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00509c7e0, 0xc00cfb98f0, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc003b4ed20) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1453 +0x1a5 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1 | |
| goroutine 7041 [select, 39 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc0000e6380, 0xc00227eea0, 0x59223f8, 0xc006a453b0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:363 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:362 +0x965 | |
| goroutine 6082 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f06905c3668, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000e23998, 0x72, 0x700, 0x773, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000e23980, 0xc001a4a800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000e23980, 0xc001a4a800, 0x773, 0x773, 0xc001a4a805, 0xc001a81198, 0xc006438828) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc002a661b0, 0xc001a4a800, 0x773, 0x773, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0016f24e0, 0xc001a4a800, 0x773, 0x773, 0x428e5b, 0xc01270ae58, 0x18) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x11c | |
| crypto/tls.(*atLeastReader).Read(0xc01270ae58, 0xc001a4a800, 0x773, 0x773, 0x76e, 0xc000382000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:776 +0x63 | |
| bytes.(*Buffer).ReadFrom(0xc00693a978, 0x58a0580, 0xc01270ae58, 0x426be5, 0x3da8720, 0x40cf8e0) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xbe | |
| crypto/tls.(*Conn).readFromUntil(0xc00693a700, 0x7f06919ac748, 0xc0016f24e0, 0x5, 0xc0016f24e0, 0x11) | |
| /usr/local/go/src/crypto/tls/conn.go:798 +0xf3 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00693a700, 0x0, 0x0, 0xa9c5be) | |
| /usr/local/go/src/crypto/tls/conn.go:605 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:573 | |
| crypto/tls.(*Conn).Read(0xc00693a700, 0xc0021de000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1276 +0x165 | |
| bufio.(*Reader).Read(0xc004681ec0, 0xc00544c118, 0x9, 0x9, 0x14, 0x7f06bcad8f18, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x58a03a0, 0xc004681ec0, 0xc00544c118, 0x9, 0x9, 0x9, 0xa7fa45, 0xc012686624, 0xc00003e510) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00544c118, 0x9, 0x9, 0x58a03a0, 0xc004681ec0, 0x0, 0x0, 0xc012686618, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00544c0e0, 0xc012686618, 0x79b3520, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0067ff200, 0xc00132b560, 0x49b0468) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:541 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0029bc1c0, 0x5981b58, 0xc0067ff200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:925 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0029bc1c0, 0x5981b58, 0xc0067ff200, 0xc006961630, 0xf) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:875 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:874 +0x5f4 | |
| goroutine 5905 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0067ff200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1061 +0x265 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:313 +0x11d5 | |
| goroutine 5904 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00220d1d0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:406 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc004681f20, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:533 +0x1dd | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2(0xc0067ff200) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:304 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:301 +0x11b0 | |
| goroutine 820052 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1.1(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/stats/automatic_stats.go:272 +0xf9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000d48d80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc02886a960) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7260 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0069e5ac0, 0x593db70, 0xc00090b650) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:357 +0x15f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc002513080, 0x593db70, 0xc00090b650, 0x43d650a, 0x29, 0xc001b23fb0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:307 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:312 +0x17e | |
| created by time.goFunc | |
| /usr/local/go/src/time/sleep.go:180 +0x45 | |
| goroutine 820115 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1.1(0x5922388, 0xc000074108) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/stats/automatic_stats.go:272 +0xf9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc00256de80, 0x5922388, 0xc000074108, 0x0, 0x0, 0xc00652f8f0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 6648 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00509d260, 0x5922350, 0xc005776cc0, 0xc005776cc0, 0xc005a480b0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:254 +0x1b8 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00509d260, 0x5922350, 0xc005776cc0, 0x2a) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:276 +0xd4 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1(0x5922350, 0xc005776b80) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:263 +0x6d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x5922350, 0xc005776b80, 0x0, 0x0, 0xc00607ab40) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 7279 [select, 39 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc002fad1d0) | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:71 +0xb2 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:62 +0x1de | |
| goroutine 7278 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc001e26788, 0xc000002eb4) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc001e26778) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc001e26770, 0x5922350, 0xc006bf33c0, 0x2ebe, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:602 +0xa5 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1(0x59223f8, 0xc001ca68a0) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:802 +0x3b4 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2(0xc000e3e380, 0x59223f8, 0xc001ca68a0, 0x0, 0x0, 0xc005f375d8) | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:442 +0xf3 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:434 +0x22b | |
| goroutine 827810 [select]: | |
| github.com/cockr |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment