Created
September 1, 2016 03:25
-
-
Save tamird/07fc7fe6e07832e6a3fc078d3603784a to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| panic: timed out during shutdown [recovered] | |
| panic: timed out during shutdown [recovered] | |
| panic: timed out during shutdown | |
| goroutine 14233 [running]: | |
| panic(0x159cec0, 0xc420eebbd0) | |
| /usr/local/go/src/runtime/panic.go:500 +0x1a1 | |
| testing.tRunner.func1(0xc4211400c0) | |
| /usr/local/go/src/testing/testing.go:579 +0x25d | |
| panic(0x159cec0, 0xc420eebbd0) | |
| /usr/local/go/src/runtime/panic.go:458 +0x243 | |
| github.com/cockroachdb/cockroach/util/leaktest.AfterTest.func1() | |
| /go/src/github.com/cockroachdb/cockroach/util/leaktest/leaktest.go:72 +0x45c | |
| panic(0x159cec0, 0xc420eebbd0) | |
| /usr/local/go/src/runtime/panic.go:458 +0x243 | |
| github.com/cockroachdb/cockroach/storage_test.(*multiTestContext).Stop(0xc4212364e0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/client_test.go:358 +0x243 | |
| github.com/cockroachdb/cockroach/storage_test.TestStoreRangeRebalance(0xc4211400c0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/client_raft_test.go:1822 +0xaca | |
| testing.tRunner(0xc4211400c0, 0x18c7380) | |
| /usr/local/go/src/testing/testing.go:610 +0x81 | |
| created by testing.(*T).Run | |
| /usr/local/go/src/testing/testing.go:646 +0x2ec | |
| goroutine 1 [chan receive]: | |
| testing.(*T).Run(0xc420084240, 0x175473d, 0x17, 0x18c7380, 0xc420279a01) | |
| /usr/local/go/src/testing/testing.go:647 +0x316 | |
| testing.RunTests.func1(0xc420084240) | |
| /usr/local/go/src/testing/testing.go:793 +0x6d | |
| testing.tRunner(0xc420084240, 0xc420287bc0) | |
| /usr/local/go/src/testing/testing.go:610 +0x81 | |
| testing.RunTests(0x18c85e8, 0x20fb500, 0x13c, 0x13c, 0xc42007db50) | |
| /usr/local/go/src/testing/testing.go:799 +0x2f5 | |
| testing.(*M).Run(0xc420287ee8, 0xc420279d30) | |
| /usr/local/go/src/testing/testing.go:743 +0x85 | |
| github.com/cockroachdb/cockroach/storage_test.TestMain(0xc420287ee8) | |
| /go/src/github.com/cockroachdb/cockroach/storage/main_test.go:57 +0x1ff | |
| main.main() | |
| github.com/cockroachdb/cockroach/storage/_test/_testmain.go:694 +0xc6 | |
| goroutine 17 [syscall, locked to thread]: | |
| runtime.goexit() | |
| /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1 | |
| goroutine 6 [chan receive]: | |
| github.com/cockroachdb/cockroach/util/log.(*loggingT).flushDaemon(0x24582e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/log/clog.go:996 +0x77 | |
| created by github.com/cockroachdb/cockroach/util/log.init.1 | |
| /go/src/github.com/cockroachdb/cockroach/util/log/clog.go:577 +0x95 | |
| goroutine 14373 [select]: | |
| github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0xe03 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420140660) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14278 [select]: | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc4210de8f0) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xa63 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm() | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x2a | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc420730110) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14336 [select]: | |
| google.golang.org/grpc/transport.(*http2Client).controller(0xc420550e00) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78 | |
| goroutine 14235 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc420d62540) | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x44 | |
| created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0x6eb | |
| goroutine 14663 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc420d66240, 0x21197c0, 0xc42115c380, 0xc42044bca8, 0xc42044bcd0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc420d66240, 0x2115a40, 0xc42061f5f0, 0xc420eca000, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc42072fa40, 0xc4207686c0, 0x20f04c0, 0xc420f7dfb0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc42072fa40, 0xc420f7dfb0) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421030050, 0xc42115e000, 0x2119a60, 0xc420b88000, 0xc42072fa40) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14438 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be04e8, 0x72, 0x12) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc421214b50, 0x72, 0xc4202f3b08, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc421214b50, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc421214af0, 0xc42040a000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc42007e1d8, 0xc42040a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc4203bcb40) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc4203bcb40, 0xc4211263f8, 0x9, 0x9, 0xc4202f3ca8, 0x627240, 0xc420964900) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc4203bcb40, 0xc4211263f8, 0x9, 0x9, 0x9, 0xc420adcc10, 0xc420adcc08, 0x21035c0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc4203bcb40, 0xc4211263f8, 0x9, 0x9, 0xc4202f3eb0, 0xd0df00, 0xc420b6c000) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc4211263f8, 0x9, 0x9, 0x20ff580, 0xc4203bcb40, 0x0, 0xc400000000, 0xc421201a40, 0xc) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc4211263c0, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc4207abf50, 0xc420add8c0, 0xc420add8c0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Client).reader(0xc420390200) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6 | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1 | |
| goroutine 14552 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc421138080) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14483 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8bdfd68, 0x72, 0x16) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc420f3ed80, 0x72, 0xc4210bb8e8, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc420f3ed80, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc420f3ed20, 0xc420652000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc42007e370, 0xc420652000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc420f48300) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc420f48300, 0xc4211264b8, 0x9, 0x9, 0x18c1805, 0x18c1804, 0xc4212292b8) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc420f48300, 0xc4211264b8, 0x9, 0x9, 0x9, 0xc4210bbaf8, 0x627240, 0xc4210f9200) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc420f48300, 0xc4211264b8, 0x9, 0x9, 0xd04552, 0xc420d5ee24, 0xc4210bbb60) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc4211264b8, 0x9, 0x9, 0x20ff580, 0xc420f48300, 0x0, 0xc400000000, 0xc4210bbc90, 0xd1211c) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc421126480, 0x35, 0x35, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc421203d70, 0xc42062bda0, 0xc42062bda0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420b893b0, 0xc421203e60) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x200 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc420561540, 0x2119a60, 0xc420b893b0) | |
| /go/src/google.golang.org/grpc/server.go:424 +0x160 | |
| google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc420561540, 0x211a000, 0xc42007e370, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:411 +0x3da | |
| google.golang.org/grpc.(*Server).handleRawConn(0xc420561540, 0x211a000, 0xc42007e370) | |
| /go/src/google.golang.org/grpc/server.go:388 +0x470 | |
| created by google.golang.org/grpc.(*Server).Serve | |
| /go/src/google.golang.org/grpc/server.go:357 +0x1a9 | |
| goroutine 14555 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc421138180) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14249 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x2fb | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420eeb370) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14744 [select]: | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).sendToReplicas(0xc420e32750, 0x7f92c8b8f2f0, 0xc421192810, 0x2540be400, 0xb2d05e00, 0xc420eebf70, 0x1, 0xc4203d8fa0, 0x3, 0x3, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:1019 +0x9e9 | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).sendRPC(0xc420e32750, 0x7f92c8b8f2f0, 0xc421192810, 0x1, 0xc4203d8fa0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:379 +0x1c6 | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).sendSingleRange(0xc420e32750, 0x7f92c8b8f2f0, 0xc421192810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:495 +0x19f | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk.func2(0xc42128ab18, 0xc42128a9d8, 0xc42128af60, 0xc420e32750, 0xc42128af50, 0x4, 0xc420b8ca00) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:763 +0x18e | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk(0xc420e32750, 0x7f92c8b8f2f0, 0xc421192810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:764 +0x455 | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).Send(0xc420e32750, 0x7f92c8b8f2f0, 0xc421192810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:606 +0x1d6 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc4210de8f0, 0x7f92c8b8f2f0, 0xc421192810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:391 +0x5de | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).send(0xc42120cd20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:495 +0x1f8 | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).(github.com/cockroachdb/cockroach/internal/client.send)-fm(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:440 +0x58 | |
| github.com/cockroachdb/cockroach/internal/client.sendAndFill(0xc42128bbd0, 0xc420f06000, 0x0, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:422 +0x168 | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).Run(0xc42120cd20, 0xc420f06000, 0x211ae01, 0xc4210e58e0) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:440 +0x75 | |
| github.com/cockroachdb/cockroach/storage.(*intentResolver).resolveIntents.func1(0x1661900, 0xc4210e58e0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/intent_resolver.go:408 +0x3a | |
| github.com/cockroachdb/cockroach/storage.(*intentResolver).resolveIntents(0xc4211f5e80, 0x7f92c8b8f2f0, 0xc4211927b0, 0xc420e338c0, 0x1, 0x1, 0x101, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/intent_resolver.go:418 +0x5c3 | |
| github.com/cockroachdb/cockroach/storage.(*intentResolver).processIntentsAsync.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/intent_resolver.go:298 +0x207 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc420e32630, 0x1e0ad98, 0x1a, 0x132, 0xc420f42d20, 0xc420b6e680) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:272 +0xbf | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunLimitedAsyncTask | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:273 +0x1a7 | |
| goroutine 14439 [select]: | |
| google.golang.org/grpc/transport.(*http2Client).controller(0xc420390200) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78 | |
| goroutine 14260 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421235440) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14354 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc4210fe000, 0xc42121e800, 0xc420731960, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x645 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x69 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4204b72f0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14635 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420a73900, 0xc42072eb40, 0xc420cde7e0) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14307 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2778 +0x10b8 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc420730a80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14484 [select]: | |
| google.golang.org/grpc/transport.(*http2Server).controller(0xc420b893b0) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:673 +0x69c | |
| created by google.golang.org/grpc/transport.newHTTP2Server | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:139 +0x767 | |
| goroutine 14559 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc4211382a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14556 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc4211381a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14477 [select]: | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc42063f600) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xa63 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm() | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x2a | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc421031580) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14701 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420b0afc0, 0xc420b71830, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4211033b0, 0xc420b71830, 0x5, 0x5, 0x0, 0x5dee0f, 0x24601c8) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4211033b0, 0xc420b71830, 0x5, 0x5, 0x5, 0xc4210de318, 0x0, 0x1) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4211033b0, 0xc420b71830, 0x5, 0x5, 0xc4210de318, 0xc42120b140, 0x3ff0000000000000) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420b71820, 0x7fffffff, 0xc42012d520, 0x0, 0xc42103ba00, 0xc4212f5ce8, 0x627240, 0xc42113fb80) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420b71820, 0x21126c0, 0x24913a0, 0xc4211033b0, 0x0, 0x0, 0x16ab9e0, 0xc4207ab920, 0x7fffffff, 0xc420029000, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc42105ed80, 0x16ab9e0, 0xc4207ab920, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc420b63800, 0x7f92c8b8f2f0, 0xc4207ab8f0, 0xc4203ce810) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc420b63800, 0xc420b0ad40, 0xc420d66240, 0xc420020f88, 0xc420020f90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42101a540) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14327 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc42100e260) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14455 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc42120b780) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14703 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420b0b200, 0xc420b71c10, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4211034a0, 0xc420b71c10, 0x5, 0x5, 0xc420e72e94, 0xc42009adc0, 0x5) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4211034a0, 0xc420b71c10, 0x5, 0x5, 0x5, 0x18c17f1, 0xc420e72e94, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4211034a0, 0xc420b71c10, 0x5, 0x5, 0x31cc4532, 0x2456f00, 0xa330) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420b71c00, 0x7fffffff, 0x15e07a0, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ffc6623, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420b71c00, 0x21126c0, 0x24913a0, 0xc4211034a0, 0x0, 0x0, 0x16b67c0, 0xc420733a40, 0x7fffffff, 0x169dd01, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc4210f1200, 0x16b67c0, 0xc420733a40, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc420b63a70, 0xc4207ab680, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc420b63a70, 0xc4211cda40, 0xc420674f88, 0xc420674f90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42101acf0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14283 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be02a8, 0x72, 0x9) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc4211c75d0, 0x72, 0xc420d7db08, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc4211c75d0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc4211c7570, 0xc420394000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc42007e228, 0xc420394000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc420f43a40) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc420f43a40, 0xc4211406f8, 0x9, 0x9, 0xc420d7dca8, 0x627240, 0xc420c6d4c0) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc420f43a40, 0xc4211406f8, 0x9, 0x9, 0x9, 0xc420a96e50, 0xc420a96e48, 0x21035c0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc420f43a40, 0xc4211406f8, 0x9, 0x9, 0xc420d7deb0, 0xd0df00, 0xc420499950) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc4211406f8, 0x9, 0x9, 0x20ff580, 0xc420f43a40, 0x0, 0xc400000000, 0xc421031df0, 0xc) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc4211406c0, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc421033bc0, 0xc420a97680, 0xc420a97680, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Client).reader(0xc420a73900) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6 | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1 | |
| goroutine 14310 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1011 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc420730ab0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14348 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420089880, 0xecf595777, 0xc41b0d3a4b, 0x2456f00, 0xc420d66080, 0xc420e32990, 0x0, 0x5fc7b3) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x311 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc42117d860) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14557 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc421138200) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14415 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be0368, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc4212146f0, 0x72, 0xc420e11df0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc4212146f0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).accept(0xc421214690, 0x0, 0x2104380, 0xc42120bc40) | |
| /usr/local/go/src/net/fd_unix.go:419 +0x238 | |
| net.(*TCPListener).accept(0xc4200265e0, 0x6069ce, 0xc420e11ea0, 0xa20c39) | |
| /usr/local/go/src/net/tcpsock_posix.go:132 +0x2e | |
| net.(*TCPListener).Accept(0xc4200265e0, 0x18c8360, 0xc42115e0a0, 0x211a000, 0xc4200265e8) | |
| /usr/local/go/src/net/tcpsock.go:222 +0x49 | |
| google.golang.org/grpc.(*Server).Serve(0xc42115e0a0, 0x2111980, 0xc4200265e0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:348 +0x15a | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x3c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42120b580) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14422 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42117a500, 0xc421220150, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4200d81e0, 0xc421220150, 0x5, 0x5, 0x5db351, 0x15d6da0, 0xc4207aa808) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4200d81e0, 0xc421220150, 0x5, 0x5, 0x5, 0xa0a5cc, 0x7f92c8b8f2f0, 0x6203ce1b0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4200d81e0, 0xc421220150, 0x5, 0x5, 0x5fd46c, 0xc4202d4ce0, 0xc4202d4ce0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc421220140, 0x7fffffff, 0x4, 0x4, 0x65f8, 0x2, 0x2, 0x2) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc421220140, 0x21126c0, 0x24913a0, 0xc4200d81e0, 0x0, 0x0, 0x16a4900, 0xc420bde2d0, 0x7fffffff, 0xc420029000, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc4205c57a0, 0x16a4900, 0xc420bde2d0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/gossip.(*gossipGossipClient).Recv(0xc420730b70, 0xc420ffe000, 0xc420bde140, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:192 +0x62 | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2.1(0x2119dc0, 0xc420730b70, 0xc420e501c0, 0xc420ffe000, 0xa6ca1f, 0xc42044af88) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:287 +0x35 | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:295 +0x9f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42117ae00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14296 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420606300, 0xc4211b0f30, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4200d8a50, 0xc4211b0f30, 0x5, 0x5, 0x5db351, 0x15d6da0, 0xc4204b6928) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4200d8a50, 0xc4211b0f30, 0x5, 0x5, 0x5, 0xb4bb77, 0xc421032870, 0xc420be8900) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4200d8a50, 0xc4211b0f30, 0x5, 0x5, 0x5fd46c, 0xc421168ce0, 0xc421168ce0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc4211b0f20, 0x7fffffff, 0x3854fd35, 0x2456f00, 0x661b, 0x100, 0x15e07a0, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc4211b0f20, 0x21126c0, 0x24913a0, 0xc4200d8a50, 0x0, 0x0, 0x16a4900, 0xc420bf8870, 0x7fffffff, 0xc420028800, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc420ed3320, 0x16a4900, 0xc420bf8870, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/gossip.(*gossipGossipClient).Recv(0xc421031610, 0xc420136f00, 0xc420bf8820, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:192 +0x62 | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2.1(0x2119dc0, 0xc421031610, 0xc4202802a0, 0xc420136f00, 0xa6ca1f, 0xc420451f88) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:287 +0x35 | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:295 +0x9f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420606400) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14472 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc420480840) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14279 [chan receive]: | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x6f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc421033860) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14487 [select]: | |
| google.golang.org/grpc.(*addrConn).transportMonitor(0xc4207a2780) | |
| /go/src/google.golang.org/grpc/clientconn.go:720 +0x63e | |
| created by google.golang.org/grpc.(*ClientConn).resetAddrConn | |
| /go/src/google.golang.org/grpc/clientconn.go:467 +0x6e2 | |
| goroutine 14363 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1031 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc4201401f0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14458 [select]: | |
| google.golang.org/grpc/transport.(*http2Server).controller(0xc420e32e10) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:673 +0x69c | |
| created by google.golang.org/grpc/transport.newHTTP2Server | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:139 +0x767 | |
| goroutine 14380 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip(0xc420280700, 0x7f92c8b8f330, 0xc4211c0bc0, 0xc420137800, 0x2119dc0, 0xc4201415d0, 0xc420e32090, 0xc420141570, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:299 +0x3c4 | |
| github.com/cockroachdb/cockroach/gossip.(*client).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:126 +0x52d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211c0ac0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14678 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc420d66240, 0x7f9200000003, 0xc4205a40c0, 0xc420f03f80, 0x2119e80, 0xc420f4f930, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc4207855f0, 0xc420f03fc0, 0x0, 0xc420fad1e0, 0x0) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc420d66240, 0xc400000003, 0xc4205a40c0, 0xc420f03f80) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420f20c90) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14486 [select]: | |
| google.golang.org/grpc/transport.(*http2Client).controller(0xc420391200) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78 | |
| goroutine 14475 [select]: | |
| github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0xe03 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc421030c60) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14349 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc42117dc20) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14440 [select]: | |
| google.golang.org/grpc.(*addrConn).transportMonitor(0xc421176780) | |
| /go/src/google.golang.org/grpc/clientconn.go:720 +0x63e | |
| created by google.golang.org/grpc.(*ClientConn).resetAddrConn | |
| /go/src/google.golang.org/grpc/clientconn.go:467 +0x6e2 | |
| goroutine 14496 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1011 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc4211ca490) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14303 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420606b80, 0xc4211b18f0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4200d8d20, 0xc4211b18f0, 0x5, 0x5, 0x2456f00, 0x185da, 0x100) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4200d8d20, 0xc4211b18f0, 0x5, 0x5, 0x5, 0xc4ffffffff, 0x0, 0xc4200855c0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4200d8d20, 0xc4211b18f0, 0x5, 0x5, 0xc400000001, 0x0, 0x5e0fb9) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc4211b18e0, 0x7fffffff, 0x0, 0x38, 0x18c1846, 0x18c1845, 0xc41ff76dad, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc4211b18e0, 0x21126c0, 0x24913a0, 0xc4200d8d20, 0x0, 0x0, 0x16b5720, 0xc421124a00, 0x7fffffff, 0x20001, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc420b88fc0, 0x16b5720, 0xc421124a00, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/gossip.(*gossipGossipServer).Recv(0xc421031890, 0x0, 0x0, 0x100010000) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:228 +0x62 | |
| github.com/cockroachdb/cockroach/gossip.(Gossip_GossipServer).Recv-fm(0xc420e50018, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x2f | |
| github.com/cockroachdb/cockroach/gossip.(*server).gossipReceiver(0xc420e50000, 0xc420026450, 0xc420606c80, 0xc420e55f40, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:310 +0x63a | |
| github.com/cockroachdb/cockroach/gossip.(*server).Gossip.func3.1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x87 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420606cc0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14396 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2778 +0x10b8 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc420141ad0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14372 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc42113b440) | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x44 | |
| created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0x6eb | |
| goroutine 14523 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc420754900) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14553 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc4211380a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14489 [chan receive]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x66 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc421160f80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14845 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420d12540, 0xc420d5edd0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc421102a50, 0xc420d5edd0, 0x5, 0x5, 0x16a02c0, 0xc420e13b01, 0x631163) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc421102a50, 0xc420d5edd0, 0x5, 0x5, 0x5, 0xc420e13c88, 0x38, 0x18c183d) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc421102a50, 0xc420d5edd0, 0x5, 0x5, 0x619e76, 0x19f75c0, 0xa4e3d) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420d5edc0, 0x7fffffff, 0xc420e13d30, 0x5d9fbf, 0x19f75c0, 0x684771, 0x2, 0x2) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420d5edc0, 0x21126c0, 0x24913a0, 0xc421102a50, 0x0, 0x0, 0x16ab9e0, 0xc4206adda0, 0x7fffffff, 0x2458fc0, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc4201dd320, 0x16ab9e0, 0xc4206adda0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc420e9f4d0, 0x5fd46c, 0xc420e13ef8, 0xc420e13ef8) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc420e9f4d0, 0xc420d12340, 0xc420d66240, 0xc420e13f88, 0xc420e13f90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4206add70) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14353 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc42117dce0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14520 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc420754900) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14568 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc421138e00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14387 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4211c18c0, 0xc42117d4b0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4200d91d0, 0xc42117d4b0, 0x5, 0x5, 0x2456f00, 0x1c5cd, 0x100) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4200d91d0, 0xc42117d4b0, 0x5, 0x5, 0x5, 0xc4ffffffff, 0x0, 0xc421126240) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4200d91d0, 0xc42117d4b0, 0x5, 0x5, 0xc400000001, 0x0, 0x5e0fb9) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc42117d4a0, 0x7fffffff, 0x0, 0x38, 0x18c1846, 0x18c1845, 0xc41ffa7a2f, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc42117d4a0, 0x21126c0, 0x24913a0, 0xc4200d91d0, 0x0, 0x0, 0x16b5720, 0xc420b0b9c0, 0x7fffffff, 0x20001, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc420b89b90, 0x16b5720, 0xc420b0b9c0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/gossip.(*gossipGossipServer).Recv(0xc4201417e0, 0x0, 0x0, 0x1000000010000) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:228 +0x62 | |
| github.com/cockroachdb/cockroach/gossip.(Gossip_GossipServer).Recv-fm(0xc420e50018, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x2f | |
| github.com/cockroachdb/cockroach/gossip.(*server).gossipReceiver(0xc420e50000, 0xc4200265c0, 0xc4211c1a80, 0xc420e59f40, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:310 +0x63a | |
| github.com/cockroachdb/cockroach/gossip.(*server).Gossip.func3.1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x87 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211c1ac0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14449 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1011 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc421030810) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14367 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42113ab40) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14450 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc42120b660) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14355 [chan receive]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x66 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42100eaa0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14386 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).Gossip(0xc420e50000, 0x21194c0, 0xc4201417e0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:180 +0x53c | |
| github.com/cockroachdb/cockroach/gossip._Gossip_Gossip_Handler(0x1695280, 0xc420e50000, 0x2115a40, 0xc420b89b90, 0xc4203cfdd0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:209 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc4200d91d0, 0xc420768c90, 0x20ea4a0, 0xc4203cfda0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc4200d91d0, 0xc4203cfda0) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421030050, 0xc42115e000, 0x2119a60, 0xc420b88000, 0xc4200d91d0) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14376 [chan receive]: | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x6f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4203cf350) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14302 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).Gossip(0xc420e50000, 0x21194c0, 0xc421031890, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:180 +0x53c | |
| github.com/cockroachdb/cockroach/gossip._Gossip_Gossip_Handler(0x1695280, 0xc420e50000, 0x2115a40, 0xc420b88fc0, 0xc4204b6360, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:209 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc4200d8d20, 0xc420768c90, 0x20ea4a0, 0xc4204b6330, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc4200d8d20, 0xc4204b6330) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421030050, 0xc42115e000, 0x2119a60, 0xc420b88000, 0xc4200d8d20) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14340 [select]: | |
| google.golang.org/grpc/transport.(*http2Server).controller(0xc420e32b40) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:673 +0x69c | |
| created by google.golang.org/grpc/transport.newHTTP2Server | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:139 +0x767 | |
| goroutine 14562 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc421138360) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14495 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).Start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:935 +0x165 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc421161040) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14771 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42117be80, 0xc420e45430, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4204983c0, 0xc420e45430, 0x5, 0x5, 0x16d8f20, 0x211a000, 0xc4200265e8) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4204983c0, 0xc420e45430, 0x5, 0x5, 0x5, 0x5d9f15, 0xc42044f458, 0x2) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4204983c0, 0xc420e45430, 0x5, 0x5, 0xa6af70, 0x2, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420e45420, 0x7fffffff, 0x50, 0x48, 0x1700360, 0x2112f00, 0xc4204e65d0, 0x48) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420e45420, 0x21126c0, 0x24913a0, 0xc4204983c0, 0x0, 0x0, 0x16ab9e0, 0xc42122e330, 0x7fffffff, 0xc4200ae800, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc4207a7440, 0x16ab9e0, 0xc42122e330, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc420eebcb0, 0x5fd46c, 0xc42044f6f8, 0xc42044f6f8) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc420eebcb0, 0xc42117a440, 0xc4211d1e80, 0xc42044f788, 0xc42044f790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42122e2d0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14479 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8bdfee8, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc420f3e530, 0x72, 0xc4204515f0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc420f3e530, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).accept(0xc420f3e4d0, 0x0, 0x2104380, 0xc421160e40) | |
| /usr/local/go/src/net/fd_unix.go:419 +0x238 | |
| net.(*TCPListener).accept(0xc42007e358, 0x6069ce, 0xc4204516a0, 0xa20c39) | |
| /usr/local/go/src/net/tcpsock_posix.go:132 +0x2e | |
| net.(*TCPListener).Accept(0xc42007e358, 0x18c8360, 0xc420561540, 0x211a000, 0xc42007e370) | |
| /usr/local/go/src/net/tcpsock.go:222 +0x49 | |
| google.golang.org/grpc.(*Server).Serve(0xc420561540, 0x2111980, 0xc42007e358, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:348 +0x15a | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x3c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc421160b80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14269 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc4211b0340) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14384 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be07e8, 0x72, 0x10) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc4210639c0, 0x72, 0xc4210b98e8, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc4210639c0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc421063960, 0xc4209d0000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc4200265b8, 0xc4209d0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc4211c3440) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc4211c3440, 0xc421126b78, 0x9, 0x9, 0x18c1805, 0x18c1804, 0xc42118fcd8) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc4211c3440, 0xc421126b78, 0x9, 0x9, 0x9, 0xc4210b9af8, 0x627240, 0xc4211c1600) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc4211c3440, 0xc421126b78, 0x9, 0x9, 0xd04552, 0xc421194684, 0xc4210b9b60) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc421126b78, 0x9, 0x9, 0x20ff580, 0xc4211c3440, 0x0, 0xc400000000, 0xc4210b9c90, 0xd1211c) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc421126b40, 0x35, 0x35, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc4203cf9e0, 0xc42062bb00, 0xc42062bb00, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420b899e0, 0xc4203cfa70) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x200 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc420f0b7c0, 0x2119a60, 0xc420b899e0) | |
| /go/src/google.golang.org/grpc/server.go:424 +0x160 | |
| google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc420f0b7c0, 0x211a000, 0xc4200265b8, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:411 +0x3da | |
| google.golang.org/grpc.(*Server).handleRawConn(0xc420f0b7c0, 0x211a000, 0xc4200265b8) | |
| /go/src/google.golang.org/grpc/server.go:388 +0x470 | |
| created by google.golang.org/grpc.(*Server).Serve | |
| /go/src/google.golang.org/grpc/server.go:357 +0x1a9 | |
| goroutine 14320 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be0a28, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc421105410, 0x72, 0xc4202c6df0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc421105410, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).accept(0xc4211053b0, 0x0, 0x2104380, 0xc420e45e00) | |
| /usr/local/go/src/net/fd_unix.go:419 +0x238 | |
| net.(*TCPListener).accept(0xc42007e2b0, 0x6069ce, 0xc4202c6ea0, 0xa20c39) | |
| /usr/local/go/src/net/tcpsock_posix.go:132 +0x2e | |
| net.(*TCPListener).Accept(0xc42007e2b0, 0x18c8360, 0xc42115f680, 0x211a000, 0xc42007e2b8) | |
| /usr/local/go/src/net/tcpsock.go:222 +0x49 | |
| google.golang.org/grpc.(*Server).Serve(0xc42115f680, 0x2111980, 0xc42007e2b0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:348 +0x15a | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x3c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420e45bc0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14705 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420c0f300, 0xc421103680, 0xc42105f680) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14569 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420d20b80, 0xecf595777, 0xc41c02ab6a, 0x2456f00, 0xc420d66080, 0xc420b88cf0, 0x0, 0x5fc7b3) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x311 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc421138ea0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14309 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).Start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:935 +0x165 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc420e445c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14636 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42103a100, 0xc420f5e330, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc42072eb40, 0xc420f5e330, 0x5, 0x5, 0x0, 0x5dee0f, 0x24601c8) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc42072eb40, 0xc420f5e330, 0x5, 0x5, 0x5, 0xc4210de318, 0x0, 0x1) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc42072eb40, 0xc420f5e330, 0x5, 0x5, 0xc4210de318, 0xc42120b140, 0x3ff0000000000000) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420f5e320, 0x7fffffff, 0xc4201dbba0, 0x0, 0xc4210ad740, 0xc42086fcf8, 0x627240, 0xc42103b840) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420f5e320, 0x21126c0, 0x24913a0, 0xc42072eb40, 0x0, 0x0, 0x16ab9e0, 0xc4207abce0, 0x7fffffff, 0xc420029000, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc420cde7e0, 0x16ab9e0, 0xc4207abce0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc42111f590, 0x7f92c8b8f2f0, 0xc4207abcb0, 0xc420fd7d40) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc42111f590, 0xc420bf7f40, 0xc420d66240, 0xc420672788, 0xc420672790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42104c420) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14273 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc4210ff480, 0xecf595777, 0xc419f72b7b, 0x2456f00, 0xc420d66080, 0xc420e321b0, 0xc4201bc000, 0xc420023ec8) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1.1(0xc4201bc000, 0xc420769170) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:259 +0x7c | |
| github.com/cockroachdb/cockroach/storage.(*storeRangeSet).Visit(0xc420769230, 0xc420c6c400) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:243 +0x21b | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:261 +0x233 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc4211b0420) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14485 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8bdfe28, 0x72, 0x15) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc420f3e7d0, 0x72, 0xc420827b08, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc420f3e7d0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc420f3e770, 0xc420662000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc42007e378, 0xc420662000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc420f48780) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc420f48780, 0xc421126578, 0x9, 0x9, 0xc420827ca8, 0x627240, 0xc420f44700) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc420f48780, 0xc421126578, 0x9, 0x9, 0x9, 0xc420616970, 0xc420616968, 0x21035c0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc420f48780, 0xc421126578, 0x9, 0x9, 0xc420827eb0, 0xd0df00, 0xc4211c40f0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc421126578, 0x9, 0x9, 0x20ff580, 0xc420f48780, 0x0, 0xc400000000, 0xc421201984, 0xc) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc421126540, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc421203f50, 0xc420add5f0, 0xc420add5f0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Client).reader(0xc420391200) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6 | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1 | |
| goroutine 14413 [select]: | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc4210de4d0) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xa63 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm() | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x2a | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc420eea9e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14871 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc420d66240, 0x21197c0, 0xc42115d230, 0xc420dd34a8, 0xc420dd34d0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc420d66240, 0x2115a40, 0xc4205af5f0, 0xc420bfc870, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc420b6c5a0, 0xc4207686c0, 0x20f04c0, 0xc420bfc810, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc420b6c5a0, 0xc420bfc810) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421030050, 0xc42115e000, 0x2119a60, 0xc420b88000, 0xc420b6c5a0) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14639 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc420d66f40, 0xc400000001, 0xc421228f60, 0xc42103a2c0, 0x2119e80, 0xc420541d20, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc42061e240, 0xc42113f780, 0x0, 0xc4202c56d8, 0x8a6222) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc420d66f40, 0xc400000001, 0xc421228f60, 0xc42103a2c0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42104cc00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14412 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:348 +0x3da | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc42120acc0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14409 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42118f620) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14298 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc4210d91a0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14490 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x420 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211ca340) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14270 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc4211b0380) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14400 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1031 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc420141b10) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14753 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42117b7c0, 0xc420e45230, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4204982d0, 0xc420e45230, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4204982d0, 0xc420e45230, 0x5, 0x5, 0x5, 0xa69a64, 0xc420678c00, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4204982d0, 0xc420e45230, 0x5, 0x5, 0x0, 0x0, 0xc420678d28) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420e45220, 0x7fffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420e45220, 0x21126c0, 0x24913a0, 0xc4204982d0, 0x0, 0x0, 0x16ab9e0, 0xc42122e120, 0x7fffffff, 0xc4200ae800, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc4207a7320, 0x16ab9e0, 0xc42122e120, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc420eebac0, 0x5fd46c, 0xc420678ef8, 0xc420678ef8) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc420eebac0, 0xc42117a380, 0xc4211d1e80, 0xc420678f88, 0xc420678f90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42122e090) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14393 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x420 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4201419d0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14750 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc4211d1e80, 0xc400000002, 0xc420f42ba0, 0xc42117a380, 0x2119e80, 0xc420eebac0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc420f601b0, 0xc42117b400, 0x0, 0xc420511a00, 0x0) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc4211d1e80, 0xc400000002, 0xc420f42ba0, 0xc42117a380) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4203cfd10) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14476 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:348 +0x3da | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc4211605c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14752 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420a73900, 0xc4204982d0, 0xc4207a7320) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14238 [select]: | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc4210de000) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xa63 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm() | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x2a | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc420eea780) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14389 [select]: | |
| google.golang.org/grpc/transport.(*http2Client).controller(0xc420551e00) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78 | |
| goroutine 14872 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4201c2440, 0xc420ecc710, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc420b6c5a0, 0xc420ecc710, 0x5, 0x5, 0x6c1be1, 0xc420b99630, 0x1) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc420b6c5a0, 0xc420ecc710, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc420b6c5a0, 0xc420ecc710, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420ecc700, 0x7fffffff, 0xc4201bc000, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ff9fd9b, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420ecc700, 0x21126c0, 0x24913a0, 0xc420b6c5a0, 0x0, 0x0, 0x16b67c0, 0xc420c04b40, 0x7fffffff, 0x1, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc4205af5f0, 0x16b67c0, 0xc420c04b40, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc42115d230, 0x7f92c8b8f2f0, 0xc420bfc870, 0xc420cee000) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc42115d230, 0xc420d66240, 0xc420dd3788, 0xc420dd3790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420bfc8d0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14299 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc4210d91a0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14433 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc4212219e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14333 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420ebfe80, 0xecf595777, 0xc41a4b5a2e, 0x2456f00, 0xc420d66080, 0xc420e32630, 0x0, 0x5fc7b3) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x311 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc42100e420) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14328 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc42100e320) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14622 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420c0f300, 0xc420498c30, 0xc4205fb320) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14391 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc4210fe000, 0xc420551c00, 0xc4201412c0, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x645 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x69 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4204e60c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14868 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4201c2100, 0xc420ecc3f0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc420b6c4b0, 0xc420ecc3f0, 0x5, 0x5, 0xc41ff6fa0d, 0x0, 0x18c19a2) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc420b6c4b0, 0xc420ecc3f0, 0x5, 0x5, 0x5, 0x100000001, 0x0, 0xc420dd3690) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc420b6c4b0, 0xc420ecc3f0, 0x5, 0x5, 0x6, 0x6, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420ecc3e0, 0x7fffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420ecc3e0, 0x21126c0, 0x24913a0, 0xc420b6c4b0, 0x0, 0x0, 0x16ab9e0, 0xc420bfc000, 0x7fffffff, 0xc420028800, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc42014c480, 0x16ab9e0, 0xc420bfc000, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc42115cf20, 0x5fd46c, 0xc420dd36f8, 0xc420dd36f8) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc42115cf20, 0xc420d12680, 0xc4210f8100, 0xc420dd3788, 0xc420dd3790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420e25f80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14339 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be0128, 0x72, 0xd) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc421105720, 0x72, 0xc4200398e8, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc421105720, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc4211056c0, 0xc4206f6000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc42007e2b8, 0xc4206f6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc42118e840) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc42118e840, 0xc4211407b8, 0x9, 0x9, 0x18c1805, 0x18c1804, 0xc4205a44d8) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc42118e840, 0xc4211407b8, 0x9, 0x9, 0x9, 0xc420039af8, 0x627240, 0xc420f44500) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc42118e840, 0xc4211407b8, 0x9, 0x9, 0xd04552, 0xc420d16fc4, 0xc420039b60) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc4211407b8, 0x9, 0x9, 0x20ff580, 0xc42118e840, 0x0, 0xc400000000, 0xc420039c90, 0xd1211c) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc421140780, 0x35, 0x35, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc420fd77d0, 0xc42062bef0, 0xc42062bef0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420e32b40, 0xc420fd7860) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x200 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc42115f680, 0x2119a60, 0xc420e32b40) | |
| /go/src/google.golang.org/grpc/server.go:424 +0x160 | |
| google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc42115f680, 0x211a000, 0xc42007e2b8, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:411 +0x3da | |
| google.golang.org/grpc.(*Server).handleRawConn(0xc42115f680, 0x211a000, 0xc42007e2b8) | |
| /go/src/google.golang.org/grpc/server.go:388 +0x470 | |
| created by google.golang.org/grpc.(*Server).Serve | |
| /go/src/google.golang.org/grpc/server.go:357 +0x1a9 | |
| goroutine 14243 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be01e8, 0x72, 0x6) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc420c9f640, 0x72, 0xc420ca3b08, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc420c9f640, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc420c9f5e0, 0xc420c54000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc42007e190, 0xc420c54000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc42120f3e0) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc42120f3e0, 0xc4211403f8, 0x9, 0x9, 0x627240, 0xc421008580, 0xc420ca3d08) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc42120f3e0, 0xc4211403f8, 0x9, 0x9, 0x9, 0xc42105dc28, 0x21035c0, 0xc42062afc0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc42120f3e0, 0xc4211403f8, 0x9, 0x9, 0xd0ce98, 0xc420498c30, 0xc420e98c00) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc4211403f8, 0x9, 0x9, 0x20ff580, 0xc42120f3e0, 0x0, 0xc400000000, 0x0, 0x56) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc4211403c0, 0x2108fc0, 0xc42062af90, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc420769bc0, 0xc42062af90, 0xc42062af90, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Client).reader(0xc420c0f300) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6 | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1 | |
| goroutine 14637 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc420d66f40, 0x21197c0, 0xc42111f620, 0xc420dccca8, 0xc420dcccd0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc420d66f40, 0x2115a40, 0xc420785e60, 0xc42104c720, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e960, 0x2119a60, 0xc420b88d80, 0xc42072ec30, 0xc4210326c0, 0x20f04c0, 0xc42104c6f0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e960, 0x2119a60, 0xc420b88d80, 0xc42072ec30, 0xc42104c6f0) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4210313e0, 0xc42115e960, 0x2119a60, 0xc420b88d80, 0xc42072ec30) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14315 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc420480180) | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x44 | |
| created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0x6eb | |
| goroutine 14295 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420c0f300, 0xc4200d8a50, 0xc420ed3320) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14423 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).Gossip(0xc420e50000, 0x21194c0, 0xc420730e70, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:180 +0x53c | |
| github.com/cockroachdb/cockroach/gossip._Gossip_Gossip_Handler(0x1695280, 0xc420e50000, 0x2115a40, 0xc420b886c0, 0xc420769b30, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:209 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc4200d82d0, 0xc420768c90, 0x20ea4a0, 0xc4207699e0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc4200d82d0, 0xc4207699e0) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421030050, 0xc42115e000, 0x2119a60, 0xc420b88000, 0xc4200d82d0) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14261 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421235440) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14250 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*bookie).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x217 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc4211f4320) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14700 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420390200, 0xc4211033b0, 0xc42105ed80) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14470 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc420480840) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14563 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc421138400) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14738 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420b0b800, 0xc4205a60d0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc421103680, 0xc4205a60d0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc421103680, 0xc4205a60d0, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc421103680, 0xc4205a60d0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc4205a60c0, 0x7fffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc4205a60c0, 0x21126c0, 0x24913a0, 0xc421103680, 0x0, 0x0, 0x16ab9e0, 0xc42101b3b0, 0x7fffffff, 0xc4200ae400, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc42105f680, 0x16ab9e0, 0xc42101b3b0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc420b63d90, 0x5fd46c, 0xc4206746f8, 0xc4206746f8) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc420b63d90, 0xc420b0b2c0, 0xc4211cda40, 0xc420674788, 0xc420674790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42101b350) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14625 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42113fa00, 0xc421022990, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc420498d20, 0xc421022990, 0x5, 0x5, 0xc420b2c594, 0xc4206b1540, 0x5) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc420498d20, 0xc421022990, 0x5, 0x5, 0x5, 0x18c17f1, 0xc420b2c594, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc420498d20, 0xc421022990, 0x5, 0x5, 0x2c1e549c, 0x2456f00, 0xe96f) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc421022980, 0x7fffffff, 0x15e07a0, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ff4ab4b, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc421022980, 0x21126c0, 0x24913a0, 0xc420498d20, 0x0, 0x0, 0x16b67c0, 0xc4216a9540, 0x7fffffff, 0x169dd01, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc4202f1710, 0x16b67c0, 0xc4216a9540, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc420541db0, 0xc420617740, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc420541db0, 0xc420d66240, 0xc4202c5788, 0xc4202c5790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc421024030) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14474 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc420480f00) | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x44 | |
| created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0x6eb | |
| goroutine 14306 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*bookie).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x217 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc420e444e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14554 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc4211380c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14294 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip(0xc4202802a0, 0x7f92c8b8f330, 0xc4206061c0, 0xc420136f00, 0x2119dc0, 0xc421031610, 0xc420e32090, 0xc4210315b0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:299 +0x3c4 | |
| github.com/cockroachdb/cockroach/gossip.(*client).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:126 +0x52d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420606080) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14411 [select]: | |
| github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0xe03 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420141dc0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14347 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc42117d840) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14394 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x2fb | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4201419e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14434 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc42016b780, 0xecf595777, 0xc41b2ec89d, 0x2456f00, 0xc420d66080, 0xc420b89710, 0x0, 0x5fc7b3) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x311 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc421221a00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14664 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421004580, 0xc420a21bd0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc42072fa40, 0xc420a21bd0, 0x5, 0x5, 0x6c1be1, 0xc420b99630, 0x1) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc42072fa40, 0xc420a21bd0, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc42072fa40, 0xc420a21bd0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420a21bc0, 0x7fffffff, 0xc4201bc000, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ff9fda5, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420a21bc0, 0x21126c0, 0x24913a0, 0xc42072fa40, 0x0, 0x0, 0x16b67c0, 0xc420c04a00, 0x7fffffff, 0x1, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc42061f5f0, 0x16b67c0, 0xc420c04a00, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc42115c380, 0x7f92c8b8f2f0, 0xc420eca000, 0xc4216a9e00) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc42115c380, 0xc420d66240, 0xc42083a788, 0xc42083a790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420eca030) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14456 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc42016b700, 0xecf595777, 0xc41b398197, 0x2456f00, 0xc420d66080, 0xc420b89710, 0x0, 0x5fc7b3) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x311 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc42120b7c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14704 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc4211cda40, 0xc400000001, 0xc420c2d020, 0xc420b0b2c0, 0x2119e80, 0xc420b63d90, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc4210f1320, 0xc420b0b740, 0x0, 0xc42027f6c0, 0x0) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc4211cda40, 0xc400000001, 0xc420c2d020, 0xc420b0b2c0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42101b0e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14419 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420089800, 0xecf595777, 0xc41b1601a8, 0x2456f00, 0xc420d66080, 0xc420e32990, 0x0, 0x5fc7b3) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x311 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc42117dd20) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14565 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc421138460) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14420 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420c0f300, 0xc4200d81e0, 0xc4205c57a0) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14247 [chan receive]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x66 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211f4300) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14866 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc4210f8100, 0xc400000001, 0xc420b052c0, 0xc420d12680, 0x2119e80, 0xc42115cf20, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc4205af560, 0xc421005e80, 0x0, 0xc420e106d8, 0x8a6222) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc4210f8100, 0xc400000001, 0xc420b052c0, 0xc420d12680) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420e25dd0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14576 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).Gossip(0xc420e50460, 0x21194c0, 0xc4211d4370, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:180 +0x53c | |
| github.com/cockroachdb/cockroach/gossip._Gossip_Gossip_Handler(0x1695280, 0xc420e50460, 0x2115a40, 0xc420e33b90, 0xc421153aa0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:209 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115f680, 0x2119a60, 0xc420e32b40, 0xc4201b90e0, 0xc420fd69f0, 0x20ea4a0, 0xc421153a40, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115f680, 0x2119a60, 0xc420e32b40, 0xc4201b90e0, 0xc421153a40) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420731a50, 0xc42115f680, 0x2119a60, 0xc420e32b40, 0xc4201b90e0) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14558 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420d20b00, 0xecf595777, 0xc41bf79d1c, 0x2456f00, 0xc420d66080, 0xc420b88cf0, 0x0, 0x5fc7b3) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x311 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc421138220) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14410 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc42118fb60) | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x44 | |
| created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0x6eb | |
| goroutine 14560 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420139080, 0xecf595777, 0xc41bf81aa5, 0x2456f00, 0xc420d66080, 0xc420e32c60, 0x0, 0x5fc7b3) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x311 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc4211382c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14454 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc42120b760) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14478 [chan receive]: | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x6f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc421203620) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14337 [select]: | |
| google.golang.org/grpc.(*addrConn).transportMonitor(0xc421177e00) | |
| /go/src/google.golang.org/grpc/clientconn.go:720 +0x63e | |
| created by google.golang.org/grpc.(*ClientConn).resetAddrConn | |
| /go/src/google.golang.org/grpc/clientconn.go:467 +0x6e2 | |
| goroutine 14571 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).Gossip(0xc420e502a0, 0x21194c0, 0xc421201bf0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:180 +0x53c | |
| github.com/cockroachdb/cockroach/gossip._Gossip_Gossip_Handler(0x1695280, 0xc420e502a0, 0x2115a40, 0xc420e33950, 0xc421152b70, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:209 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e960, 0x2119a60, 0xc420b88d80, 0xc4201b8f00, 0xc421032cc0, 0x20ea4a0, 0xc421152b40, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e960, 0x2119a60, 0xc420b88d80, 0xc4201b8f00, 0xc421152b40) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4210313e0, 0xc42115e960, 0x2119a60, 0xc420b88d80, 0xc4201b8f00) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14590 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4210ace00, 0xc420b42bb0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4201b8ff0, 0xc420b42bb0, 0x5, 0x5, 0x5db351, 0x15d6da0, 0xc4211d2d18) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4201b8ff0, 0xc420b42bb0, 0x5, 0x5, 0x5, 0xa0a5cc, 0x7f92c8b8f2f0, 0x5204b7140) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4201b8ff0, 0xc420b42bb0, 0x5, 0x5, 0x5fd46c, 0xc420448ce0, 0xc420448ce0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420b42ba0, 0x7fffffff, 0x4, 0x4, 0x1a70f2, 0x4, 0x4, 0x4) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420b42ba0, 0x21126c0, 0x24913a0, 0xc4201b8ff0, 0x0, 0x0, 0x16a4900, 0xc420bde8c0, 0x7fffffff, 0xc420029000, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc420c6b560, 0x16a4900, 0xc420bde8c0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/gossip.(*gossipGossipClient).Recv(0xc4211d4210, 0xc420d0c000, 0xc42023c640, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:192 +0x62 | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2.1(0x2119dc0, 0xc4211d4210, 0xc420e50d20, 0xc420d0c000, 0xa6ca1f, 0xc420c4d788) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:287 +0x35 | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:295 +0x9f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4210addc0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14488 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc4210fe000, 0xc420391000, 0xc4211ca080, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x645 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x69 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211fc030) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14357 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x2fb | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4201400c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14280 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be0068, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc4211c73a0, 0x72, 0xc42044c5f0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc4211c73a0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).accept(0xc4211c7340, 0x0, 0x2104380, 0xc4211b0620) | |
| /usr/local/go/src/net/fd_unix.go:419 +0x238 | |
| net.(*TCPListener).accept(0xc42007e220, 0x6069ce, 0xc42044c6a0, 0xa20c39) | |
| /usr/local/go/src/net/tcpsock_posix.go:132 +0x2e | |
| net.(*TCPListener).Accept(0xc42007e220, 0x18c8360, 0xc42115e960, 0x211a000, 0xc420026430) | |
| /usr/local/go/src/net/tcpsock.go:222 +0x49 | |
| google.golang.org/grpc.(*Server).Serve(0xc42115e960, 0x2111980, 0xc42007e220, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:348 +0x15a | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x3c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420e44320) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14493 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2778 +0x10b8 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc4211ca460) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14388 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be08a8, 0x72, 0xf) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc421063560, 0x72, 0xc4209ccb08, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc421063560, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc421063500, 0xc4209e0000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc4200265c8, 0xc4209e0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc4211c3f20) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc4211c3f20, 0xc421126c38, 0x9, 0x9, 0xc4209ccca8, 0x627240, 0xc4211c1d80) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc4211c3f20, 0xc421126c38, 0x9, 0x9, 0x9, 0xc4206160d0, 0xc4206160c8, 0x21035c0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc4211c3f20, 0xc421126c38, 0x9, 0x9, 0xc4209cceb0, 0xd0df00, 0xc420499d10) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc421126c38, 0x9, 0x9, 0x20ff580, 0xc4211c3f20, 0x0, 0xc400000000, 0xc420eeb600, 0xc) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc421126c00, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc4204e6030, 0xc4206165d0, 0xc4206165d0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Client).reader(0xc420551e00) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6 | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1 | |
| goroutine 14239 [chan receive]: | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x6f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420769890) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14368 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42113ab40) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14277 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:348 +0x3da | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc4211f59c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14751 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc4211d1e80, 0xc400000005, 0xc420f431a0, 0xc42117a440, 0x2119e80, 0xc420eebcb0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc420f60240, 0xc42117bd40, 0x0, 0xc42057e340, 0x0) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc4211d1e80, 0xc400000005, 0xc420f431a0, 0xc42117a440) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4203cfd40) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14574 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip(0xc420e50d20, 0x7f92c8b8f330, 0xc4210accc0, 0xc420d0c000, 0x2119dc0, 0xc4211d4210, 0xc420e32090, 0xc4211d41b0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:299 +0x3c4 | |
| github.com/cockroachdb/cockroach/gossip.(*client).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:126 +0x52d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4210acc80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14358 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*bookie).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x217 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc42100eac0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14457 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8bdffa8, 0x72, 0x13) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc4212155d0, 0x72, 0xc420dbc8e8, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc4212155d0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc421215570, 0xc42041a000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc4200265e8, 0xc42041a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc420f424e0) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc420f424e0, 0xc421140278, 0x9, 0x9, 0x18c1805, 0x18c1804, 0xc421235c18) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc420f424e0, 0xc421140278, 0x9, 0x9, 0x9, 0xc420dbcaf8, 0x627240, 0xc420c6c280) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc420f424e0, 0xc421140278, 0x9, 0x9, 0xd04552, 0xc4211b1524, 0xc420dbcb60) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc421140278, 0x9, 0x9, 0x20ff580, 0xc420f424e0, 0x0, 0xc400000000, 0xc420dbcc90, 0xd1211c) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc421140240, 0x35, 0x35, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc421032300, 0xc42062b800, 0xc42062b800, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420e32e10, 0xc421032ae0) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x200 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc42115e0a0, 0x2119a60, 0xc420e32e10) | |
| /go/src/google.golang.org/grpc/server.go:424 +0x160 | |
| google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc42115e0a0, 0x211a000, 0xc4200265e8, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:411 +0x3da | |
| google.golang.org/grpc.(*Server).handleRawConn(0xc42115e0a0, 0x211a000, 0xc4200265e8) | |
| /go/src/google.golang.org/grpc/server.go:388 +0x470 | |
| created by google.golang.org/grpc.(*Server).Serve | |
| /go/src/google.golang.org/grpc/server.go:357 +0x1a9 | |
| goroutine 14385 [select]: | |
| google.golang.org/grpc/transport.(*http2Server).controller(0xc420b899e0) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:673 +0x69c | |
| created by google.golang.org/grpc/transport.newHTTP2Server | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:139 +0x767 | |
| goroutine 14623 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42113f900, 0xc4210225f0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc420498c30, 0xc4210225f0, 0x5, 0x5, 0xc4208b5bc0, 0xc4208b5b98, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc420498c30, 0xc4210225f0, 0x5, 0x5, 0x5, 0xa10ca3, 0x24582e0, 0xc400000003) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc420498c30, 0xc4210225f0, 0x5, 0x5, 0x58, 0x9d5, 0x10) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc4210225e0, 0x7fffffff, 0x2c, 0xc4208b5e28, 0x3, 0x3, 0xc4208b5d20, 0xa0a9fe) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc4210225e0, 0x21126c0, 0x24913a0, 0xc420498c30, 0x0, 0x0, 0x16ab9e0, 0xc42062b0b0, 0x7fffffff, 0xc4200ae800, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc4205fb320, 0x16ab9e0, 0xc42062b0b0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc420541d20, 0x7f92c8b8f2f0, 0xc42062b050, 0xc420a96de0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc420541d20, 0xc42103a2c0, 0xc420d66f40, 0xc42083af88, 0xc42083af90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42105dcb0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14564 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc421138440) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14362 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1011 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc4201401e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14867 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420c0f300, 0xc420b6c4b0, 0xc42014c480) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14255 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1031 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc420eeb890) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14566 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc4211384c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14699 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc420d66240, 0x7f9200000005, 0xc4205cff80, 0xc420b0ad40, 0x2119e80, 0xc420b63800, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc4210f0630, 0xc420b0aec0, 0x0, 0xc42012d520, 0x0) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc420d66240, 0xc400000005, 0xc4205cff80, 0xc420b0ad40) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42101a2a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14399 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1011 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc420141b00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14325 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc42100e220) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14577 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4210ad240, 0xc420b42d90, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4201b90e0, 0xc420b42d90, 0x5, 0x5, 0x2456f00, 0x18675, 0x100) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4201b90e0, 0xc420b42d90, 0x5, 0x5, 0x5, 0xc4ffffffff, 0x0, 0xc420ba4240) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4201b90e0, 0xc420b42d90, 0x5, 0x5, 0xc400000001, 0x0, 0x5e0fb9) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420b42d80, 0x7fffffff, 0x0, 0x38, 0x18c1846, 0x18c1845, 0xc41ffa7a15, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420b42d80, 0x21126c0, 0x24913a0, 0xc4201b90e0, 0x0, 0x0, 0x16b5720, 0xc420b0bd00, 0x7fffffff, 0x20001, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc420e33b90, 0x16b5720, 0xc420b0bd00, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/gossip.(*gossipGossipServer).Recv(0xc4211d4370, 0x0, 0x0, 0x100010000) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:228 +0x62 | |
| github.com/cockroachdb/cockroach/gossip.(Gossip_GossipServer).Recv-fm(0xc420e50478, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x2f | |
| github.com/cockroachdb/cockroach/gossip.(*server).gossipReceiver(0xc420e50460, 0xc4200267a8, 0xc4210ad380, 0xc420875f40, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:310 +0x63a | |
| github.com/cockroachdb/cockroach/gossip.(*server).Gossip.func3.1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x87 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4210ad540) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14271 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc4211b03a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14297 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc4210d91a0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14300 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc4210d91a0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14540 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4210e06c0, 0xc421147bb0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4201b8a50, 0xc421147bb0, 0x5, 0x5, 0x5db351, 0x15d6da0, 0xc4211d2418) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4201b8a50, 0xc421147bb0, 0x5, 0x5, 0x5, 0xa0a5cc, 0x7f92c8b8f2f0, 0x6204e6cc0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4201b8a50, 0xc421147bb0, 0x5, 0x5, 0x5fd46c, 0xc420330ce0, 0xc420330ce0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc421147ba0, 0x7fffffff, 0x4, 0x4, 0x16b4d1, 0x1, 0x1, 0x6) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc421147ba0, 0x21126c0, 0x24913a0, 0xc4201b8a50, 0x0, 0x0, 0x16a4900, 0xc420bdfe00, 0x7fffffff, 0x2458fc0, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc420c6ad80, 0x16a4900, 0xc420bdfe00, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/gossip.(*gossipGossipClient).Recv(0xc421200a00, 0xc420ffe780, 0xc420bded70, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:192 +0x62 | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2.1(0x2119dc0, 0xc421200a00, 0xc420e50b60, 0xc420ffe780, 0xa6ca1f, 0xc420c4e788) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:287 +0x35 | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:295 +0x9f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4210e08c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14275 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc420b51620) | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x44 | |
| created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0x6eb | |
| goroutine 14289 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x2fb | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420730940) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14446 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2778 +0x10b8 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc4210307d0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14390 [select]: | |
| google.golang.org/grpc.(*addrConn).transportMonitor(0xc4210017c0) | |
| /go/src/google.golang.org/grpc/clientconn.go:720 +0x63e | |
| created by google.golang.org/grpc.(*ClientConn).resetAddrConn | |
| /go/src/google.golang.org/grpc/clientconn.go:467 +0x6e2 | |
| goroutine 14335 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be0728, 0x72, 0xc) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc4211056b0, 0x72, 0xc420035b08, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc4211056b0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc421105650, 0xc4202dc000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc4200264c8, 0xc4202dc000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc4203bdf80) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc4203bdf80, 0xc4211269f8, 0x9, 0x9, 0xc420035ca8, 0x627240, 0xc4210f8b80) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc4203bdf80, 0xc4211269f8, 0x9, 0x9, 0x9, 0xc420616040, 0xc420616038, 0x21035c0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc4203bdf80, 0xc4211269f8, 0x9, 0x9, 0xc420035eb0, 0xd0df00, 0xc420499c20) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc4211269f8, 0x9, 0x9, 0x20ff580, 0xc4203bdf80, 0x0, 0xc400000000, 0xc420140534, 0xc) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc4211269c0, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc4204b7260, 0xc420adca80, 0xc420adca80, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Client).reader(0xc420550e00) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6 | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1 | |
| goroutine 14770 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420390200, 0xc4204983c0, 0xc4207a7440) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14567 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420139000, 0xecf595777, 0xc41bf8a473, 0x2456f00, 0xc420d66080, 0xc420e32c60, 0x0, 0x5fc7b3) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x311 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc4211384e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14290 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc4211b04c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14285 [select]: | |
| google.golang.org/grpc.(*addrConn).transportMonitor(0xc421177540) | |
| /go/src/google.golang.org/grpc/clientconn.go:720 +0x63e | |
| created by google.golang.org/grpc.(*ClientConn).resetAddrConn | |
| /go/src/google.golang.org/grpc/clientconn.go:467 +0x6e2 | |
| goroutine 14522 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc420754900) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14572 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4210aca00, 0xc420b42130, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4201b8f00, 0xc420b42130, 0x5, 0x5, 0x2456f00, 0x1bb886, 0x100) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4201b8f00, 0xc420b42130, 0x5, 0x5, 0x5, 0xc4ffffffff, 0x0, 0xc420085680) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4201b8f00, 0xc420b42130, 0x5, 0x5, 0xc400000001, 0x0, 0x5e0fb9) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420b42120, 0x7fffffff, 0x0, 0x38, 0x18c1846, 0x18c1845, 0xc41ff97607, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420b42120, 0x21126c0, 0x24913a0, 0xc4201b8f00, 0x0, 0x0, 0x16b5720, 0xc420d13ec0, 0x7fffffff, 0x20001, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc420e33950, 0x16b5720, 0xc420d13ec0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/gossip.(*gossipGossipServer).Recv(0xc421201bf0, 0x0, 0x0, 0x1000000010000) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:228 +0x62 | |
| github.com/cockroachdb/cockroach/gossip.(Gossip_GossipServer).Recv-fm(0xc420e502b8, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x2f | |
| github.com/cockroachdb/cockroach/gossip.(*server).gossipReceiver(0xc420e502a0, 0xc420026788, 0xc4210acb40, 0xc420833f40, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:310 +0x63a | |
| github.com/cockroachdb/cockroach/gossip.(*server).Gossip.func3.1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x87 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4210acb80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14473 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc420480840) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14739 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc420d66240, 0x21197c0, 0xc420b295b0, 0xc4202c84a8, 0xc4202c84d0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc420d66240, 0x2115a40, 0xc420cf7170, 0xc420b31110, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc421103950, 0xc4207686c0, 0x20f04c0, 0xc420b310e0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc421103950, 0xc420b310e0) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421030050, 0xc42115e000, 0x2119a60, 0xc420b88000, 0xc421103950) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14258 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421235440) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14624 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc420d66240, 0x21197c0, 0xc420541db0, 0xc4202c54a8, 0xc4202c54d0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc420d66240, 0x2115a40, 0xc4202f1710, 0xc421024000, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc420498d20, 0xc4207686c0, 0x20f04c0, 0xc42105dfb0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc420498d20, 0xc42105dfb0) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421030050, 0xc42115e000, 0x2119a60, 0xc420b88000, 0xc420498d20) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14374 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:348 +0x3da | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc42117c0e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14245 [select]: | |
| google.golang.org/grpc.(*addrConn).transportMonitor(0xc421176c80) | |
| /go/src/google.golang.org/grpc/clientconn.go:720 +0x63e | |
| created by google.golang.org/grpc.(*ClientConn).resetAddrConn | |
| /go/src/google.golang.org/grpc/clientconn.go:467 +0x6e2 | |
| goroutine 14414 [chan receive]: | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x6f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4204e7d70) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14575 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420550e00, 0xc4201b8ff0, 0xc420c6b560) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14491 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x2fb | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211ca350) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14350 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc42117dc40) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14452 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc42120b6e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14406 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42118f620) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14251 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2778 +0x10b8 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc420eeb840) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14679 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420550e00, 0xc4211c44b0, 0xc42029d7a0) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14407 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42118f620) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14326 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc42100e240) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14443 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x420 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4210306c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14361 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).Start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:935 +0x165 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc42100eb60) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14234 [chan receive]: | |
| github.com/cockroachdb/cockroach/rpc.NewContext.func1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:126 +0x76 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4212202c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14222 [select]: | |
| google.golang.org/grpc/transport.(*http2Server).controller(0xc420b88000) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:673 +0x69c | |
| created by google.golang.org/grpc/transport.newHTTP2Server | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:139 +0x767 | |
| goroutine 14253 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).Start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:935 +0x165 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc4211f4a00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14445 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*bookie).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x217 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc4211f4980) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14330 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc42100e360) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14529 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip(0xc420e50b60, 0x7f92c8b8f330, 0xc4210e0600, 0xc420ffe780, 0x2119dc0, 0xc421200a00, 0xc420e32090, 0xc4212009a0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:299 +0x3c4 | |
| github.com/cockroachdb/cockroach/gossip.(*client).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:126 +0x52d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4210e05c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14369 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42113ab40) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14244 [select]: | |
| google.golang.org/grpc/transport.(*http2Client).controller(0xc420c0f300) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78 | |
| goroutine 14287 [chan receive]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x66 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420e444c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14293 [select]: | |
| google.golang.org/grpc/transport.(*http2Server).controller(0xc420b88d80) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:673 +0x69c | |
| created by google.golang.org/grpc/transport.newHTTP2Server | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:139 +0x767 | |
| goroutine 14466 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1031 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc421030820) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14292 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be05a8, 0x72, 0xa) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc421215e90, 0x72, 0xc420d828e8, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc421215e90, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc421215e30, 0xc420f9e000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc420026430, 0xc420f9e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc4210d87e0) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc4210d87e0, 0xc421126878, 0x9, 0x9, 0x18c1805, 0x18c1804, 0xc42118eb38) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc4210d87e0, 0xc421126878, 0x9, 0x9, 0x9, 0xc420d82af8, 0x627240, 0xc420c6d480) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc4210d87e0, 0xc421126878, 0x9, 0x9, 0xd04552, 0xc42100e8e4, 0xc420d82b60) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc421126878, 0x9, 0x9, 0x20ff580, 0xc4210d87e0, 0x0, 0xc400000000, 0xc420d82c90, 0xd1211c) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc421126840, 0x35, 0x35, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc4207ab710, 0xc42062b950, 0xc42062b950, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420b88d80, 0xc4207ab7d0) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x200 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc42115e960, 0x2119a60, 0xc420b88d80) | |
| /go/src/google.golang.org/grpc/server.go:424 +0x160 | |
| google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc42115e960, 0x211a000, 0xc420026430, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:411 +0x3da | |
| google.golang.org/grpc.(*Server).handleRawConn(0xc42115e960, 0x211a000, 0xc420026430) | |
| /go/src/google.golang.org/grpc/server.go:388 +0x470 | |
| created by google.golang.org/grpc.(*Server).Serve | |
| /go/src/google.golang.org/grpc/server.go:357 +0x1a9 | |
| goroutine 14561 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc421138320) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14268 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc4211b0320) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14395 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*bookie).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x217 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc42117d6e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14530 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420a73900, 0xc4201b8a50, 0xc420c6ad80) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14521 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc420754900) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14398 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).Start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:935 +0x165 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc42117d820) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14375 [select]: | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc42063ee70) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xa63 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm() | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x2a | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc420140c60) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14351 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc42117dc60) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14359 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2778 +0x10b8 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc4201401b0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14317 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:348 +0x3da | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc420e45380) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14319 [chan receive]: | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x6f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420fd7530) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14381 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420c0f300, 0xc4200d90e0, 0xc4205c4900) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14236 [select]: | |
| github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0xe03 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420eea190) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14383 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4211c0d00, 0xc42117d050, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4200d90e0, 0xc42117d050, 0x5, 0x5, 0x5db351, 0x15d6da0, 0xc420fd7c48) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4200d90e0, 0xc42117d050, 0x5, 0x5, 0x5, 0xa0a5cc, 0x7f92c8b8f2f0, 0x520fd6330) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4200d90e0, 0xc42117d050, 0x5, 0x5, 0x5fd46c, 0xc4210bcce0, 0xc4210bcce0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc42117d040, 0x7fffffff, 0x4, 0x4, 0x11a523, 0x3, 0x3, 0x4) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc42117d040, 0x21126c0, 0x24913a0, 0xc4200d90e0, 0x0, 0x0, 0x16a4900, 0xc420bde050, 0x7fffffff, 0xc420029000, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc4205c4900, 0x16a4900, 0xc420bde050, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/gossip.(*gossipGossipClient).Recv(0xc4201415d0, 0xc420137800, 0xc420158960, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:192 +0x62 | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2.1(0x2119dc0, 0xc4201415d0, 0xc420280700, 0xc420137800, 0xa6ca1f, 0xc420e14f88) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:287 +0x35 | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:295 +0x9f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211c0e00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14272 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc4211b03e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14442 [chan receive]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x66 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211f4960) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14284 [select]: | |
| google.golang.org/grpc/transport.(*http2Client).controller(0xc420a73900) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78 | |
| goroutine 14288 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x420 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420730920) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14259 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421235440) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14444 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x2fb | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4210306d0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14267 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc4211b02e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14286 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc4210fe000, 0xc420a73700, 0xc4207307a0, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x645 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x69 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc421033d10) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14471 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc420480840) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14318 [select]: | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc4210dee70) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xa63 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm() | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x2a | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc420731330) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14497 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1031 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc4211ca4a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14254 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1011 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc420eeb880) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14492 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*bookie).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x217 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b88cf0, 0xc421160fa0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14638 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42103a280, 0xc420f5e450, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc42072ec30, 0xc420f5e450, 0x5, 0x5, 0x6c1be1, 0xc420faa4b0, 0x1) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc42072ec30, 0xc420f5e450, 0x5, 0x5, 0x5, 0x0, 0x22, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc42072ec30, 0xc420f5e450, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420f5e440, 0x7fffffff, 0xd, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ffd0173, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420f5e440, 0x21126c0, 0x24913a0, 0xc42072ec30, 0x0, 0x0, 0x16b67c0, 0xc4205fd040, 0x7fffffff, 0x1, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc420785e60, 0x16b67c0, 0xc4205fd040, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc42111f620, 0x7f92c8b8f2f0, 0xc42104c720, 0xc420a31a40) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc42111f620, 0xc420d66f40, 0xc420dccf88, 0xc420dccf90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42104c750) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14246 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc4210fe000, 0xc420c0f100, 0xc420eeb230, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x645 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x69 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420769c50) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14352 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc42117dc80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14311 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1031 +0x261 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc420730ac0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14332 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc42100e400) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14451 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc42120b6a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14453 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420b89710, 0xc42120b740) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14418 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x2cc | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32990, 0xc42117dd00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14377 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be0968, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc421063330, 0x72, 0xc4210b8de0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc421063330, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).accept(0xc4210632d0, 0x0, 0x2104380, 0xc42117d260) | |
| /usr/local/go/src/net/fd_unix.go:419 +0x238 | |
| net.(*TCPListener).accept(0xc420026590, 0x6069ce, 0xc4210b8e90, 0xa20c39) | |
| /usr/local/go/src/net/tcpsock_posix.go:132 +0x2e | |
| net.(*TCPListener).Accept(0xc420026590, 0x18c8360, 0xc420f0b7c0, 0x211a000, 0xc4200265b8) | |
| /usr/local/go/src/net/tcpsock.go:222 +0x49 | |
| google.golang.org/grpc.(*Server).Serve(0xc420f0b7c0, 0x2111980, 0xc420026590, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:348 +0x15a | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x3c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42117c8a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14331 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420ebfe00, 0xecf595777, 0xc41a472c70, 0x2456f00, 0xc420d66080, 0xc420e32630, 0x0, 0x5fc7b3) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x311 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32630, 0xc42100e380) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14291 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc4210ff500, 0xecf595777, 0xc419fa5a59, 0x2456f00, 0xc420d66080, 0xc420e321b0, 0xc4201bc000, 0xc4200246c8) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x237 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1.1(0xc4201bc000, 0xc420769170) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:259 +0x7c | |
| github.com/cockroachdb/cockroach/storage.(*storeRangeSet).Visit(0xc420769620, 0xc420c6cb80) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:243 +0x21b | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:261 +0x233 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc4211b04e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14424 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42117b0c0, 0xc421220e70, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4200d82d0, 0xc421220e70, 0x5, 0x5, 0x2456f00, 0x15fb5, 0x100) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4200d82d0, 0xc421220e70, 0x5, 0x5, 0x5, 0xc4ffffffff, 0x0, 0xc420b2cc00) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4200d82d0, 0xc421220e70, 0x5, 0x5, 0xc400000001, 0x0, 0x5e0fb9) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc421220e60, 0x7fffffff, 0x0, 0x38, 0x18c1846, 0x18c1845, 0xc41ffa7a07, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc421220e60, 0x21126c0, 0x24913a0, 0xc4200d82d0, 0x0, 0x0, 0x16b5720, 0xc420b0bec0, 0x7fffffff, 0x20001, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc420b886c0, 0x16b5720, 0xc420b0bec0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/gossip.(*gossipGossipServer).Recv(0xc420730e70, 0x0, 0x0, 0x1000000010000) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:228 +0x62 | |
| github.com/cockroachdb/cockroach/gossip.(Gossip_GossipServer).Recv-fm(0xc420e50018, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x2f | |
| github.com/cockroachdb/cockroach/gossip.(*server).gossipReceiver(0xc420e50000, 0xc42007e110, 0xc42117b280, 0xc421129f40, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:310 +0x63a | |
| github.com/cockroachdb/cockroach/gossip.(*server).Gossip.func3.1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x87 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42117b300) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14237 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:348 +0x3da | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc421220e40) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14408 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42118f620) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14634 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc420d66240, 0x7f9200000002, 0xc421228840, 0xc420bf7f40, 0x2119e80, 0xc42111f590, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc420784510, 0xc420bf7f80, 0x0, 0xc420facb60, 0x0) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc420d66240, 0xc400000002, 0xc421228840, 0xc420bf7f40) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42104c2d0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14405 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*client).gossip(0xc420e501c0, 0x7f92c8b8f330, 0xc4211ccd80, 0xc420ffe000, 0x2119dc0, 0xc420730b70, 0xc420e32090, 0xc420141cb0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:299 +0x3c4 | |
| github.com/cockroachdb/cockroach/gossip.(*client).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/client.go:126 +0x52d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211ccc80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14221 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be0428, 0x72, 0x7) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc4212140d0, 0x72, 0xc4200378e8, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc4212140d0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).Read(0xc421214070, 0xc421148000, 0x8000, 0x8000, 0x0, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1a1 | |
| net.(*conn).Read(0xc420026000, 0xc421148000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:173 +0x70 | |
| bufio.(*Reader).fill(0xc421234180) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x10c | |
| bufio.(*Reader).Read(0xc421234180, 0xc4211260f8, 0x9, 0x9, 0x18c1805, 0x18c1804, 0xc4203b7a98) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x1bc | |
| io.ReadAtLeast(0x20ff580, 0xc421234180, 0xc4211260f8, 0x9, 0x9, 0x9, 0xc420037af8, 0x627240, 0xc420b4bd00) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x20ff580, 0xc421234180, 0xc4211260f8, 0x9, 0x9, 0xd04552, 0xc420a21b04, 0xc420037b60) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| golang.org/x/net/http2.readFrameHeader(0xc4211260f8, 0x9, 0x9, 0x20ff580, 0xc421234180, 0x0, 0xc400000000, 0xc420037c90, 0xd1211c) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0x7b | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc4211260c0, 0x35, 0x35, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0xa4 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc4207aa060, 0xc4209600f0, 0xc4209600f0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f | |
| google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420b88000, 0xc4207aa0f0) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x200 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc42115e000, 0x2119a60, 0xc420b88000) | |
| /go/src/google.golang.org/grpc/server.go:424 +0x160 | |
| google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc42115e000, 0x211a000, 0xc420026000, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:411 +0x3da | |
| google.golang.org/grpc.(*Server).handleRawConn(0xc42115e000, 0x211a000, 0xc420026000) | |
| /go/src/google.golang.org/grpc/server.go:388 +0x470 | |
| created by google.golang.org/grpc.(*Server).Serve | |
| /go/src/google.golang.org/grpc/server.go:357 +0x1a9 | |
| goroutine 14366 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42113ab40) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x4a | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xa7 | |
| goroutine 14248 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x420 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420eeb360) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14316 [select]: | |
| github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0xe03 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420730d00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14714 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc420d66240, 0x7f9200000004, 0xc420d03620, 0xc421125480, 0x2119e80, 0xc420b8f4f0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc42073ea20, 0xc4211254c0, 0x0, 0xc420021f80, 0x0) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc420d66240, 0xc400000004, 0xc420d03620, 0xc421125480) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420be6de0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14240 [IO wait]: | |
| net.runtime_pollWait(0x7f92c8be0668, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x59 | |
| net.(*pollDesc).wait(0xc420c9f410, 0x72, 0xc420038de0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x38 | |
| net.(*pollDesc).waitRead(0xc420c9f410, 0x2107000, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x34 | |
| net.(*netFD).accept(0xc420c9f3b0, 0x0, 0x2104380, 0xc42120a100) | |
| /usr/local/go/src/net/fd_unix.go:419 +0x238 | |
| net.(*TCPListener).accept(0xc42007e188, 0x6069ce, 0xc420038e90, 0xa20c39) | |
| /usr/local/go/src/net/tcpsock_posix.go:132 +0x2e | |
| net.(*TCPListener).Accept(0xc42007e188, 0x18c8360, 0xc42115e000, 0x211a000, 0xc420026000) | |
| /usr/local/go/src/net/tcpsock.go:222 +0x49 | |
| google.golang.org/grpc.(*Server).Serve(0xc42115e000, 0x2111980, 0xc42007e188, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:348 +0x15a | |
| github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x3c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211f4080) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14276 [select]: | |
| github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0xe03 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420eebb00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14356 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x420 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4201400b0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14441 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc4210fe000, 0xc42121eb00, 0xc420eeb0d0, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x645 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x69 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4204b6030) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14448 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).Start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:935 +0x165 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32c60, 0xc4211f4a80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14570 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*idAllocator).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/id_alloc.go:124 +0x7ef | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e321b0, 0xc421201810) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14392 [chan receive]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x66 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc42117d6a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14680 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420c24100, 0xc420d16f70, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4211c44b0, 0xc420d16f70, 0x5, 0x5, 0xc42120fdf8, 0x1, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4211c44b0, 0xc420d16f70, 0x5, 0x5, 0x5, 0xc420e277a8, 0x0, 0xc420f20189) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4211c44b0, 0xc420d16f70, 0x5, 0x5, 0x2, 0xc42083ac70, 0x5e0779) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420d16f60, 0x7fffffff, 0xc42083ad10, 0x8b98a7, 0xc420f20185, 0x25, 0x2b, 0x210cc40) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420d16f60, 0x21126c0, 0x24913a0, 0xc4211c44b0, 0x0, 0x0, 0x16ab9e0, 0xc420f20e40, 0x7fffffff, 0xc4200ae800, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc42029d7a0, 0x16ab9e0, 0xc420f20e40, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc420f4f930, 0x2a, 0x30, 0x1470066ee4a84222) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc420f4f930, 0xc420f03f80, 0xc420d66240, 0xc42083af88, 0xc42083af90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420f20e10) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14681 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc420d67800, 0x21197c0, 0xc420f4f9b0, 0xc420451ca8, 0xc420451cd0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc420d67800, 0x2115a40, 0xc42008fb90, 0xc420f21170, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115f680, 0x2119a60, 0xc420e32b40, 0xc4211c45a0, 0xc420fd6210, 0x20f04c0, 0xc420f21140, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115f680, 0x2119a60, 0xc420e32b40, 0xc4211c45a0, 0xc420f21140) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420731a50, 0xc42115f680, 0x2119a60, 0xc420e32b40, 0xc4211c45a0) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14682 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420c241c0, 0xc420d17070, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4211c45a0, 0xc420d17070, 0x5, 0x5, 0x6c1be1, 0xc420ea6eb0, 0x1) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4211c45a0, 0xc420d17070, 0x5, 0x5, 0x5, 0x0, 0x22, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4211c45a0, 0xc420d17070, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420d17060, 0x7fffffff, 0xc4205d6600, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ff9fdb9, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420d17060, 0x21126c0, 0x24913a0, 0xc4211c45a0, 0x0, 0x0, 0x16b67c0, 0xc420c04780, 0x7fffffff, 0x1, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc42008fb90, 0x16b67c0, 0xc420c04780, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc420f4f9b0, 0x7f92c8b8f2f0, 0xc420f21170, 0xc4216a9b80) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc420f4f9b0, 0xc420d67800, 0xc420451f88, 0xc420451f90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420f211a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14683 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc420d67800, 0xc400000001, 0xc4205a46c0, 0xc420c24400, 0x2119e80, 0xc420f4fca0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc42008fcb0, 0xc420c24780, 0x0, 0xc42044e6d8, 0x8a6222) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc420d67800, 0xc400000001, 0xc4205a46c0, 0xc420c24400) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420f21530) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14684 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420c0f300, 0xc4211c4780, 0xc42023a240) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14685 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420c24880, 0xc420d17430, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc4211c4780, 0xc420d17430, 0x5, 0x5, 0x671698, 0x15923c0, 0x1) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc4211c4780, 0xc420d17430, 0x5, 0x5, 0x5, 0xc420f4f9b0, 0x0, 0xc4200ae800) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc4211c4780, 0xc420d17430, 0x5, 0x5, 0xc420451c68, 0x5e2ce8, 0x10) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420d17420, 0x7fffffff, 0xa1d6af, 0x2103480, 0xc4211c45a0, 0xc420d17070, 0x5, 0x5) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420d17420, 0x21126c0, 0x24913a0, 0xc4211c4780, 0x0, 0x0, 0x16ab9e0, 0xc420f216e0, 0x7fffffff, 0xc4200ae800, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc42023a240, 0x16ab9e0, 0xc420f216e0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc420f4fca0, 0xc420c37a40, 0xc420451ed8, 0x6679d2) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc420f4fca0, 0xc420c24400, 0xc420d67800, 0xc420451f88, 0xc420451f90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420f21680) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14730 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420b0be40, 0xc420eac9d0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc421103950, 0xc420eac9d0, 0x5, 0x5, 0x6c1be1, 0xc420b99630, 0x2) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc421103950, 0xc420eac9d0, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc421103950, 0xc420eac9d0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420eac9c0, 0x7fffffff, 0xc4201bc300, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ffa3f2d, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420eac9c0, 0x21126c0, 0x24913a0, 0xc421103950, 0x0, 0x0, 0x16b67c0, 0xc420b81900, 0x7fffffff, 0x1, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc420cf7170, 0x16b67c0, 0xc420b81900, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc420b295b0, 0x7f92c8b8f2f0, 0xc420b31110, 0xc420b80500) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc420b295b0, 0xc420d66240, 0xc4202c8788, 0xc4202c8790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420b31290) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14702 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc4211cda40, 0x21197c0, 0xc420b63a70, 0xc4206754a8, 0xc4206754d0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc4211cda40, 0x2115a40, 0xc4210f1200, 0xc42101acc0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e0a0, 0x2119a60, 0xc420e32e10, 0xc4211034a0, 0xc4204e6c60, 0x20f04c0, 0xc42101ac90, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e0a0, 0x2119a60, 0xc420e32e10, 0xc4211034a0, 0xc42101ac90) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420eeb530, 0xc42115e0a0, 0x2119a60, 0xc420e32e10, 0xc4211034a0) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14715 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420551e00, 0xc420b64d20, 0xc42100bb00) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14716 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421125640, 0xc420e2d4b0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc420b64d20, 0xc420e2d4b0, 0x5, 0x5, 0x0, 0x5dee0f, 0x24601c8) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc420b64d20, 0xc420e2d4b0, 0x5, 0x5, 0x5, 0xc4210de318, 0x0, 0x1) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc420b64d20, 0xc420e2d4b0, 0x5, 0x5, 0xc4210de318, 0xc42120b140, 0x3ff0000000000000) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420e2d4a0, 0x7fffffff, 0xc4205d1ba0, 0x0, 0xc420c24b40, 0xc42086ccf8, 0x627240, 0xc420ec96c0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420e2d4a0, 0x21126c0, 0x24913a0, 0xc420b64d20, 0x0, 0x0, 0x16ab9e0, 0xc4207abe60, 0x7fffffff, 0xc420029000, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc42100bb00, 0x16ab9e0, 0xc4207abe60, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc420b8f4f0, 0x7f92c8b8f2f0, 0xc4207abe30, 0xc420fd7ef0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc420b8f4f0, 0xc421125480, 0xc420d66240, 0xc42083af88, 0xc42083af90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420be6f00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14717 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc4211d1e80, 0x21197c0, 0xc420b8f560, 0xc420021ca8, 0xc420021cd0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc4211d1e80, 0x2115a40, 0xc4206eabd0, 0xc420be73e0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc420f0b7c0, 0x2119a60, 0xc420b899e0, 0xc420b64e10, 0xc4203ce150, 0x20f04c0, 0xc420be73b0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc420f0b7c0, 0x2119a60, 0xc420b899e0, 0xc420b64e10, 0xc420be73b0) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201416c0, 0xc420f0b7c0, 0x2119a60, 0xc420b899e0, 0xc420b64e10) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14718 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421125700, 0xc420e2d610, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc420b64e10, 0xc420e2d610, 0x5, 0x5, 0xc4202467d4, 0xc42088fcc0, 0x5) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc420b64e10, 0xc420e2d610, 0x5, 0x5, 0x5, 0x18c17f1, 0xc4202467d4, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc420b64e10, 0xc420e2d610, 0x5, 0x5, 0x31c815b9, 0x2456f00, 0xd0f0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420e2d600, 0x7fffffff, 0x15e07a0, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ffc667d, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420e2d600, 0x21126c0, 0x24913a0, 0xc420b64e10, 0x0, 0x0, 0x16b67c0, 0xc420732f00, 0x7fffffff, 0x169dd01, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc4206eabd0, 0x16b67c0, 0xc420732f00, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc420b8f560, 0xc4207ab1d0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc420b8f560, 0xc4211d1e80, 0xc420021f88, 0xc420021f90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420be7410) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14719 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc4211d1e80, 0xc400000001, 0xc420d03da0, 0xc4211259c0, 0x2119e80, 0xc420b8f8b0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc4206eacf0, 0xc421125dc0, 0x0, 0xc4202c66d8, 0x8a6222) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc4211d1e80, 0xc400000001, 0xc420d03da0, 0xc4211259c0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420be7800) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14720 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420c0f300, 0xc420b64ff0, 0xc42100be60) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14721 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421125ec0, 0xc420e2da30, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc420b64ff0, 0xc420e2da30, 0x5, 0x5, 0x671698, 0x15923c0, 0x1) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc420b64ff0, 0xc420e2da30, 0x5, 0x5, 0x5, 0xc420b8f560, 0x0, 0xc420029000) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc420b64ff0, 0xc420e2da30, 0x5, 0x5, 0xc420021c68, 0x5e2ce8, 0x10) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420e2da20, 0x7fffffff, 0xa1d6af, 0x2103480, 0xc420b64e10, 0xc420e2d610, 0x5, 0x5) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420e2da20, 0x21126c0, 0x24913a0, 0xc420b64ff0, 0x0, 0x0, 0x16ab9e0, 0xc420be79b0, 0x7fffffff, 0xc420029000, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc42100be60, 0x16ab9e0, 0xc420be79b0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc420b8f8b0, 0xc42100d040, 0xc420021ed8, 0x6679d2) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc420b8f8b0, 0xc4211259c0, 0xc4211d1e80, 0xc420021f88, 0xc420021f90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420be7950) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14722 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc420d66240, 0x21197c0, 0xc420758950, 0xc4202c64a8, 0xc4202c64d0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc420d66240, 0x2115a40, 0xc4206eb4d0, 0xc420b84ed0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc420b652c0, 0xc4207686c0, 0x20f04c0, 0xc420b84ea0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e000, 0x2119a60, 0xc420b88000, 0xc420b652c0, 0xc420b84ea0) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421030050, 0xc42115e000, 0x2119a60, 0xc420b88000, 0xc420b652c0) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14725 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420ec85c0, 0xc420b6af90, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc420b652c0, 0xc420b6af90, 0x5, 0x5, 0x6c1be1, 0xc420b99630, 0x2) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc420b652c0, 0xc420b6af90, 0x5, 0x5, 0x5, 0x1, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc420b652c0, 0xc420b6af90, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420b6af80, 0x7fffffff, 0xc4201bc300, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ffc2ee1, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420b6af80, 0x21126c0, 0x24913a0, 0xc420b652c0, 0x0, 0x0, 0x16b67c0, 0xc4207a2280, 0x7fffffff, 0x1, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc4206eb4d0, 0x16b67c0, 0xc4207a2280, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc420758950, 0x7f92c8b8f2f0, 0xc420b84ed0, 0xc4206e23c0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc420758950, 0xc420d66240, 0xc4202c6788, 0xc4202c6790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420b84f00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14762 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc4211cda40, 0x21197c0, 0xc421030880, 0xc420c4cca8, 0xc420c4ccd0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc4211cda40, 0x2115a40, 0xc4210f0090, 0xc4204e6840, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e0a0, 0x2119a60, 0xc420e32e10, 0xc42072e000, 0xc4204e6c60, 0x20f04c0, 0xc4204e6780, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e0a0, 0x2119a60, 0xc420e32e10, 0xc42072e000, 0xc4204e6780) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420eeb530, 0xc42115e0a0, 0x2119a60, 0xc420e32e10, 0xc42072e000) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14763 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4210f8180, 0xc4211b1650, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc42072e000, 0xc4211b1650, 0x5, 0x5, 0x6c1be1, 0xc42071b630, 0x2) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc42072e000, 0xc4211b1650, 0x5, 0x5, 0x5, 0x0, 0x1b, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc42072e000, 0xc4211b1650, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc4211b1640, 0x7fffffff, 0xc420432300, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ff98c23, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc4211b1640, 0x21126c0, 0x24913a0, 0xc42072e000, 0x0, 0x0, 0x16b67c0, 0xc420ce7a40, 0x7fffffff, 0x1, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc4210f0090, 0x16b67c0, 0xc420ce7a40, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc421030880, 0x7f92c8b8f2f0, 0xc4204e6840, 0xc420ce7400) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc421030880, 0xc4211cda40, 0xc420c4cf88, 0xc420c4cf90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4204e6960) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14846 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc4210f8100, 0x21197c0, 0xc420e9f550, 0xc420c4d4a8, 0xc420c4d4d0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc4210f8100, 0x2115a40, 0xc420785b00, 0xc420c1a1e0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc420561540, 0x2119a60, 0xc420b893b0, 0xc421102b40, 0xc4204b7050, 0x20f04c0, 0xc420c1a1b0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc420561540, 0x2119a60, 0xc420b893b0, 0xc421102b40, 0xc420c1a1b0) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4211ca280, 0xc420561540, 0x2119a60, 0xc420b893b0, 0xc421102b40) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14847 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420d12640, 0xc420d5eed0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc421102b40, 0xc420d5eed0, 0x5, 0x5, 0x6c1be1, 0xc420a37130, 0x1) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc421102b40, 0xc420d5eed0, 0x5, 0x5, 0x5, 0x0, 0x22, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc421102b40, 0xc420d5eed0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc420d5eec0, 0x7fffffff, 0xc420604600, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ff9fdcd, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc420d5eec0, 0x21126c0, 0x24913a0, 0xc421102b40, 0x0, 0x0, 0x16b67c0, 0xc420c04500, 0x7fffffff, 0x1, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc420785b00, 0x16b67c0, 0xc420c04500, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc420e9f550, 0x7f92c8b8f2f0, 0xc420c1a1e0, 0xc4216a9900) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc420e9f550, 0xc4210f8100, 0xc420c53f88, 0xc420c53f90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc420c1a210) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14764 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc4211cda40, 0xc400000004, 0xc420480780, 0xc4210f8200, 0x2119e80, 0xc421030c70, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc4210f0120, 0xc4210f8a80, 0x0, 0xc420c36000, 0x0) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc4211cda40, 0xc400000004, 0xc420480780, 0xc4210f8200) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4204e7fb0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14844 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420391200, 0xc421102a50, 0xc4201dd320) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14843 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc420d66240, 0x7f9200000006, 0xc42063d5c0, 0xc420d12340, 0x2119e80, 0xc420e9f4d0, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc4205af050, 0xc420d12400, 0x0, 0x0, 0x2) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc420d66240, 0xc400000006, 0xc42063d5c0, 0xc420d12340) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4206adb60) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14765 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420551e00, 0xc42072e0f0, 0xc4210f58c0) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14766 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4210f9640, 0xc42100e6f0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc42072e0f0, 0xc42100e6f0, 0x5, 0x5, 0x0, 0x3, 0x3) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc42072e0f0, 0xc42100e6f0, 0x5, 0x5, 0x5, 0x0, 0x0, 0xc420c4d450) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc42072e0f0, 0xc42100e6f0, 0x5, 0x5, 0xc420c4d79e, 0xc420175680, 0xc420c4d798) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc42100e6e0, 0x7fffffff, 0x3, 0xc420c4d798, 0x3, 0x5e3e0e, 0x7f92c8bc0238, 0x8) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc42100e6e0, 0x21126c0, 0x24913a0, 0xc42072e0f0, 0x0, 0x0, 0x16ab9e0, 0xc421203260, 0x7fffffff, 0xc420029000, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc4210f58c0, 0x16ab9e0, 0xc421203260, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc421030c70, 0xa54a32, 0xc420c6b560, 0x16a4900) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc421030c70, 0xc4210f8200, 0xc4211cda40, 0xc420c4d788, 0xc420c4d790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc421202840) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14767 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc420d66f40, 0x21197c0, 0xc421030da0, 0xc420c4cca8, 0xc420c4ccd0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc420d66f40, 0x2115a40, 0xc4210f01b0, 0xc421203c50, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc42115e960, 0x2119a60, 0xc420b88d80, 0xc42072e1e0, 0xc4210326c0, 0x20f04c0, 0xc421203b30, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc42115e960, 0x2119a60, 0xc420b88d80, 0xc42072e1e0, 0xc421203b30) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4210313e0, 0xc42115e960, 0x2119a60, 0xc420b88d80, 0xc42072e1e0) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14768 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4210f9900, 0xc42100ef50, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc42072e1e0, 0xc42100ef50, 0x5, 0x5, 0x6c1be1, 0xc420faa4b0, 0x2) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc42072e1e0, 0xc42100ef50, 0x5, 0x5, 0x5, 0x0, 0x1b, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc42072e1e0, 0xc42100ef50, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc42100ef40, 0x7fffffff, 0xc4205d6c00, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ff98c0f, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc42100ef40, 0x21126c0, 0x24913a0, 0xc42072e1e0, 0x0, 0x0, 0x16b67c0, 0xc420ce7cc0, 0x7fffffff, 0x1, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc4210f01b0, 0x16b67c0, 0xc420ce7cc0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc421030da0, 0x7f92c8b8f2f0, 0xc421203c50, 0xc420ce7040) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc421030da0, 0xc420d66f40, 0xc420c4cf88, 0xc420c4cf90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc421203cb0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14769 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc420d66f40, 0xc400000004, 0xc42118ec60, 0xc4210f9cc0, 0x2119e80, 0xc421031960, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:459 +0x4b5 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:407 +0x40c | |
| github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc4210f0240, 0xc4210f9fc0, 0x0, 0xc420c36640, 0x0) | |
| /go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).connectAndProcess(0xc420d66f40, 0xc400000004, 0xc42118ec60, 0xc4210f9cc0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:408 +0x138 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).SendAsync.func2.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:538 +0x4c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211fc7e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14786 [select]: | |
| google.golang.org/grpc.NewClientStream.func2(0x2119a00, 0xc420551e00, 0xc42072e2d0, 0xc4210f5c20) | |
| /go/src/google.golang.org/grpc/stream.go:197 +0x426 | |
| created by google.golang.org/grpc.NewClientStream | |
| /go/src/google.golang.org/grpc/stream.go:217 +0xb5f | |
| goroutine 14787 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420f44240, 0xc421194270, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc42072e2d0, 0xc421194270, 0x5, 0x5, 0x0, 0x3, 0x3) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc42072e2d0, 0xc421194270, 0x5, 0x5, 0x5, 0x0, 0x0, 0xc420c4d450) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc42072e2d0, 0xc421194270, 0x5, 0x5, 0xc420c4d79e, 0xc420175680, 0xc420c4d798) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc421194260, 0x7fffffff, 0x3, 0xc420c4d798, 0x3, 0x5e3e0e, 0x7f92c8bc0238, 0x8) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc421194260, 0x21126c0, 0x24913a0, 0xc42072e2d0, 0x0, 0x0, 0x16ab9e0, 0xc4211fca50, 0x7fffffff, 0xc420029000, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc4210f5c20, 0x16ab9e0, 0xc4211fca50, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:305 +0xc4 | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).Recv(0xc421031960, 0xa54a32, 0xc420c6b560, 0x16a4900) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:233 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1.1(0x2119e80, 0xc421031960, 0xc4210f9cc0, 0xc420d66f40, 0xc420c4d788, 0xc420c4d790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:434 +0x59 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:449 +0x52 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211fc9f0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14788 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc4211d1e80, 0x21197c0, 0xc421031fc0, 0xc420c4cca8, 0xc420c4ccd0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc4211d1e80, 0x2115a40, 0xc4210f05a0, 0xc4211fd890, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc420f0b7c0, 0x2119a60, 0xc420b899e0, 0xc42072e3c0, 0xc4203ce150, 0x20f04c0, 0xc4211fd860, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc420f0b7c0, 0x2119a60, 0xc420b899e0, 0xc42072e3c0, 0xc4211fd860) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201416c0, 0xc420f0b7c0, 0x2119a60, 0xc420b899e0, 0xc42072e3c0) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14789 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc4211d1e80, 0x21197c0, 0xc421031b00, 0xc420c4d4a8, 0xc420c4d4d0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:303 +0x1b0 | |
| github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x16d07a0, 0xc4211d1e80, 0x2115a40, 0xc4210f0360, 0xc4211fd2c0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:282 +0xbb | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc420f0b7c0, 0x2119a60, 0xc420b899e0, 0xc42072e4b0, 0xc4203ce150, 0x20f04c0, 0xc4211fd290, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:689 +0x702 | |
| google.golang.org/grpc.(*Server).handleStream(0xc420f0b7c0, 0x2119a60, 0xc420b899e0, 0xc42072e4b0, 0xc4211fd290) | |
| /go/src/google.golang.org/grpc/server.go:773 +0xc36 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4201416c0, 0xc420f0b7c0, 0x2119a60, 0xc420b899e0, 0xc42072e4b0) | |
| /go/src/google.golang.org/grpc/server.go:422 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/src/google.golang.org/grpc/server.go:423 +0xa3 | |
| goroutine 14790 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420f45340, 0xc4211948b0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc42072e4b0, 0xc4211948b0, 0x5, 0x5, 0x6c1be1, 0xc420dbb3b0, 0x2) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc42072e4b0, 0xc4211948b0, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc42072e4b0, 0xc4211948b0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc4211948a0, 0x7fffffff, 0x0, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ff9fdf5, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc4211948a0, 0x21126c0, 0x24913a0, 0xc42072e4b0, 0x0, 0x0, 0x16b67c0, 0xc420c04000, 0x7fffffff, 0x1, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc4210f0360, 0x16b67c0, 0xc420c04000, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc421031b00, 0x7f92c8b8f2f0, 0xc4211fd2c0, 0xc420ce72c0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc421031b00, 0xc4211d1e80, 0xc420c4d788, 0xc420c4d790) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211fd2f0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14791 [select]: | |
| google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420f44ec0, 0xc4211958f0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a | |
| google.golang.org/grpc/transport.(*Stream).Read(0xc42072e3c0, 0xc4211958f0, 0x5, 0x5, 0x6c1be1, 0xc420dbb3b0, 0x2) | |
| /go/src/google.golang.org/grpc/transport/transport.go:314 +0x59 | |
| io.ReadAtLeast(0x2103480, 0xc42072e3c0, 0xc4211958f0, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xa4 | |
| io.ReadFull(0x2103480, 0xc42072e3c0, 0xc4211958f0, 0x5, 0x5, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x58 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc4211958e0, 0x7fffffff, 0xc4201bcf00, 0x138, 0x18c1d65, 0x18c1d64, 0xc41ff9fdeb, 0x0) | |
| /go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f | |
| google.golang.org/grpc.recv(0xc4211958e0, 0x21126c0, 0x24913a0, 0xc42072e3c0, 0x0, 0x0, 0x16b67c0, 0xc420c04140, 0x7fffffff, 0x1, ...) | |
| /go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc4210f05a0, 0x16b67c0, 0xc420c04140, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/stream.go:492 +0xec | |
| github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc421031fc0, 0x7f92c8b8f2f0, 0xc4211fd890, 0xc420ce7680) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:301 +0x62 | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x21197c0, 0xc421031fc0, 0xc4211d1e80, 0xc420c4cf88, 0xc420c4cf90) | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:280 +0x3c | |
| github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:297 +0x45 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420e32090, 0xc4211fd8c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x7d | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x66 | |
| goroutine 14796 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLease(0xc4205d6000, 0x7f92c8b8f2f0, 0xc420b499e0, 0xc4210a6f60) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:721 +0x4ec | |
| github.com/cockroachdb/cockroach/storage.(*Replica).addWriteCmd(0xc4205d6000, 0x7f92c8b8f2f0, 0xc420b499e0, 0x6b49d200, 0x63, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:1427 +0x4f5 | |
| github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc4205d6000, 0x7f92c8b8f2f0, 0xc420b499e0, 0x6b49d200, 0x63, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:1001 +0x277 | |
| github.com/cockroachdb/cockroach/storage.(*Store).Send(0xc420faa280, 0x7f92c8b8f2f0, 0xc420b49860, 0x6b49d200, 0x63, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2049 +0x733 | |
| github.com/cockroachdb/cockroach/storage.(*Stores).Send(0xc420d67400, 0x7f92c8b8f330, 0xc42103a6c0, 0x0, 0x0, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/stores.go:182 +0x202 | |
| github.com/cockroachdb/cockroach/storage_test.(*multiTestContextKVTransport).SendNext.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/client_test.go:471 +0x12e | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask.func1(0xc420e32630, 0x1e09b4e, 0x16, 0x1f5, 0xc4210a6a40) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:234 +0x95 | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:235 +0xbc | |
| goroutine 14797 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Replica).addWriteCmd(0xc4205d6000, 0x7f92c8b8f2f0, 0xc420b49bf0, 0x6b49d200, 0x66, 0x0, 0x0, 0x1, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:1457 +0x451 | |
| github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc4205d6000, 0x7f92c8b8f2f0, 0xc420b49bf0, 0x6b49d200, 0x66, 0x0, 0x0, 0x1, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:1001 +0x277 | |
| github.com/cockroachdb/cockroach/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica_range_lease.go:125 +0x170 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask.func1(0xc420e32630, 0x1e0a857, 0x1e, 0x8f, 0xc420b49b90) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:234 +0x95 | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:235 +0xbc | |
| goroutine 14912 [semacquire]: | |
| sync.runtime_Semacquire(0xc420c8325c) | |
| /usr/local/go/src/runtime/sema.go:47 +0x30 | |
| sync.(*WaitGroup).Wait(0xc420c83250) | |
| /usr/local/go/src/sync/waitgroup.go:131 +0x97 | |
| github.com/cockroachdb/cockroach/storage_test.(*multiTestContext).Stop.func1(0xc4212364e0, 0xc420f90480) | |
| /go/src/github.com/cockroachdb/cockroach/storage/client_test.go:332 +0x13a | |
| created by github.com/cockroachdb/cockroach/storage_test.(*multiTestContext).Stop | |
| /go/src/github.com/cockroachdb/cockroach/storage/client_test.go:345 +0x7c | |
| goroutine 14930 [semacquire]: | |
| sync.runtime_notifyListWait(0xc420d66f90, 0xc400000000) | |
| /usr/local/go/src/runtime/sema.go:267 +0x122 | |
| sync.(*Cond).Wait(0xc420d66f80) | |
| /usr/local/go/src/sync/cond.go:57 +0x80 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).Quiesce(0xc420e32630) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:416 +0x1ea | |
| github.com/cockroachdb/cockroach/storage_test.(*multiTestContext).Stop.func1.1(0xc420c83250, 0xc420e32630) | |
| /go/src/github.com/cockroachdb/cockroach/storage/client_test.go:328 +0x68 | |
| created by github.com/cockroachdb/cockroach/storage_test.(*multiTestContext).Stop.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/client_test.go:330 +0x101 | |
| FAIL github.com/cockroachdb/cockroach/storage 44.156s |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment