Skip to content

Instantly share code, notes, and snippets.

@tamird
Created August 30, 2016 16:57
Show Gist options
  • Save tamird/88cbc53ff744f6486813c587b6cfc08d to your computer and use it in GitHub Desktop.
Save tamird/88cbc53ff744f6486813c587b6cfc08d to your computer and use it in GitHub Desktop.
panic: test timed out after 8m0s
goroutine 383962 [running]:
panic(0x1aef200, 0xc420e174e0)
/usr/local/go/src/runtime/panic.go:500 +0x1ae
testing.startAlarm.func1()
/usr/local/go/src/testing/testing.go:918 +0x14e
created by time.goFunc
/usr/local/go/src/time/sleep.go:154 +0x78
goroutine 1 [chan receive, 6 minutes]:
testing.(*T).Run(0xc42024d740, 0x1caa9c5, 0x1f, 0x1f651b0, 0x9ad401)
/usr/local/go/src/testing/testing.go:647 +0x56e
testing.RunTests.func1(0xc42024d740)
/usr/local/go/src/testing/testing.go:793 +0xba
testing.tRunner(0xc42024d740, 0xc42024bb38)
/usr/local/go/src/testing/testing.go:610 +0xca
testing.RunTests(0x1f663b0, 0x2837260, 0x13b, 0x13b, 0xc42028e338)
/usr/local/go/src/testing/testing.go:799 +0x4bb
testing.(*M).Run(0xc42024bef0, 0xc42027bce0)
/usr/local/go/src/testing/testing.go:743 +0x130
github.com/cockroachdb/cockroach/storage_test.TestMain(0xc42024bef0)
/go/src/github.com/cockroachdb/cockroach/storage/main_test.go:57 +0x287
main.main()
github.com/cockroachdb/cockroach/storage/_test/_testmain.go:692 +0x1b6
goroutine 17 [syscall, 8 minutes, locked to thread]:
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1
goroutine 19 [chan receive]:
github.com/cockroachdb/cockroach/util/log.(*loggingT).flushDaemon(0x2b93d00)
/go/src/github.com/cockroachdb/cockroach/util/log/clog.go:980 +0x85
created by github.com/cockroachdb/cockroach/util/log.init.1
/go/src/github.com/cockroachdb/cockroach/util/log/clog.go:563 +0xc3
goroutine 17362 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7f033355b6b0, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc42166be20, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc42166be20, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).accept(0xc42166bdc0, 0x0, 0x2841f80, 0xc421965280)
/usr/local/go/src/net/fd_unix.go:419 +0x2b8
net.(*TCPListener).accept(0xc420026588, 0xc420026590, 0xc421afde98, 0x6146ef)
/usr/local/go/src/net/tcpsock_posix.go:132 +0x51
net.(*TCPListener).Accept(0xc420026588, 0x1f66120, 0xc421385d60, 0x28573c0, 0xc420026590)
/usr/local/go/src/net/tcpsock.go:222 +0x50
google.golang.org/grpc.(*Server).Serve(0xc421385d60, 0x284edc0, 0xc420026588, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:348 +0x1b8
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x54
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421965160)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17236 [IO wait]:
net.runtime_pollWait(0x7f033355a930, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc420b4f6b0, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc420b4f6b0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc420b4f650, 0xc42048a000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc420026020, 0xc42048a000, 0x8000, 0x8000, 0xc420f3d358, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc420026020, 0xc42048a000, 0x8000, 0x8000, 0x0, 0x7fd4, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc4219b2660)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc4219b2660, 0xc4219b0278, 0x9, 0x9, 0x0, 0x0, 0xc4212dfb58)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc4219b2660, 0xc4219b0278, 0x9, 0x9, 0x9, 0xc420f3ced8, 0xc420128680, 0x0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc4219b2660, 0xc4219b0278, 0x9, 0x9, 0x1, 0xc420a7b250, 0x0)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc4219b0278, 0x9, 0x9, 0x283d340, 0xc4219b2660, 0x0, 0x0, 0xc4209ca0f0, 0x0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc4219b0240, 0x0, 0x0, 0x0, 0x0)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc4213ee270, 0xc420f3d350, 0xc420f3d350, 0xc4212dfed0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Client).reader(0xc421468200)
/go/src/google.golang.org/grpc/transport/http2_client.go:921 +0x154
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:203 +0xc1a
goroutine 17864 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2745 +0x184a
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc4214aa4e0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17849 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2745 +0x184a
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42012fa00)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17888 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc42038c900)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17902 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc42038ce80)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17284 [select, 6 minutes]:
google.golang.org/grpc/transport.(*http2Server).controller(0xc420cc4480)
/go/src/google.golang.org/grpc/transport/http2_server.go:673 +0xa56
created by google.golang.org/grpc/transport.newHTTP2Server
/go/src/google.golang.org/grpc/transport/http2_server.go:139 +0xa91
goroutine 17228 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:348 +0x519
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421aeab80)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17377 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x86
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc420c3bbf0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17502 [select, 6 minutes]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421a12b00, 0xc4218a5270, 0x5, 0x5, 0x0, 0x0, 0x0)
/go/src/google.golang.org/grpc/transport/transport.go:141 +0x89c
google.golang.org/grpc/transport.(*Stream).Read(0xc4206add10, 0xc4218a5270, 0x5, 0x5, 0x61a0f7, 0xc420c12730, 0xc420c12714)
/go/src/google.golang.org/grpc/transport/transport.go:314 +0x79
io.ReadAtLeast(0x2841080, 0xc4206add10, 0xc4218a5270, 0x5, 0x5, 0x5, 0xc420faf7e8, 0x61a47a, 0xc420129380)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x2841080, 0xc4206add10, 0xc4218a5270, 0x5, 0x5, 0xc4ffffffff, 0x0, 0xc420c126c0)
/usr/local/go/src/io/io.go:325 +0x73
google.golang.org/grpc.(*parser).recvMsg(0xc4218a5260, 0x7fffffff, 0x5e25a5, 0xc4205e05f8, 0x0, 0x0, 0x6, 0x0)
/go/src/google.golang.org/grpc/rpc_util.go:231 +0xa4
google.golang.org/grpc.recv(0xc4218a5260, 0x284fac0, 0x354ff60, 0xc4206add10, 0x0, 0x0, 0x1c03280, 0xc421a67500, 0x7fffffff, 0x1, ...)
/go/src/google.golang.org/grpc/rpc_util.go:315 +0x5e
google.golang.org/grpc.(*serverStream).RecvMsg(0xc420388090, 0x1c03280, 0xc421a67500, 0x0, 0x0)
/go/src/google.golang.org/grpc/stream.go:492 +0x154
github.com/cockroachdb/cockroach/gossip.(*gossipGossipServer).Recv(0xc421179480, 0xc4ffffffff, 0x0, 0x180002)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:230 +0x86
github.com/cockroachdb/cockroach/gossip.(Gossip_GossipServer).Recv-fm(0xc42037c398, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x4b
github.com/cockroachdb/cockroach/gossip.(*server).gossipReceiver(0xc42037c380, 0xc420026780, 0xc421a12bc0, 0xc420faff40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:310 +0x604
github.com/cockroachdb/cockroach/gossip.(*server).Gossip.func3.1()
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x116
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421a12c00)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17361 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x86
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421339500)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17905 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc421796800, 0xecf57acee, 0xc41f38717a, 0x2b92920, 0xc4207dc200, 0xc421870ea0, 0x0, 0xc420f69f58)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x299
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x540
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc42038cf00)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17348 [select]:
github.com/cockroachdb/cockroach/gossip.(*client).gossip(0xc42037c460, 0x7f0333502738, 0xc421a1fc00, 0xc42081a900, 0x2857180, 0xc420d31420, 0xc420cc4090, 0xc420d313c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:299 +0x4b5
github.com/cockroachdb/cockroach/gossip.(*client).start.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:126 +0x72d
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421a1fb40)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17305 [select]:
github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc421aa8000, 0xc420409400, 0xc42090ed30, 0xf, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:282 +0x813
github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1()
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:205 +0xb3
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4213b8f30)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17878 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42038c520)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17852 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42037fd40)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17566 [select, 6 minutes]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4206b6880, 0xc4214b1770, 0x5, 0x5, 0x0, 0x0, 0x0)
/go/src/google.golang.org/grpc/transport/transport.go:141 +0x89c
google.golang.org/grpc/transport.(*Stream).Read(0xc42116f950, 0xc4214b1770, 0x5, 0x5, 0xc420afec80, 0xc420737630, 0xc420737614)
/go/src/google.golang.org/grpc/transport/transport.go:314 +0x79
io.ReadAtLeast(0x2841080, 0xc42116f950, 0xc4214b1770, 0x5, 0x5, 0x5, 0xc42142b7e8, 0x61a47a, 0xc42028b520)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x2841080, 0xc42116f950, 0xc4214b1770, 0x5, 0x5, 0xc4ffffffff, 0x0, 0xc4207375c0)
/usr/local/go/src/io/io.go:325 +0x73
google.golang.org/grpc.(*parser).recvMsg(0xc4214b1760, 0x7fffffff, 0x5f0480, 0x7f032b03e0a0, 0x11, 0x7f03334da152, 0x5f05c4, 0x7f03334da152)
/go/src/google.golang.org/grpc/rpc_util.go:231 +0xa4
google.golang.org/grpc.recv(0xc4214b1760, 0x284fac0, 0x354ff60, 0xc42116f950, 0x0, 0x0, 0x1c03280, 0xc42114c440, 0x7fffffff, 0x1, ...)
/go/src/google.golang.org/grpc/rpc_util.go:315 +0x5e
google.golang.org/grpc.(*serverStream).RecvMsg(0xc420389050, 0x1c03280, 0xc42114c440, 0x0, 0x0)
/go/src/google.golang.org/grpc/stream.go:492 +0x154
github.com/cockroachdb/cockroach/gossip.(*gossipGossipServer).Recv(0xc421a23070, 0xc4ffffffff, 0x0, 0x180002)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:230 +0x86
github.com/cockroachdb/cockroach/gossip.(Gossip_GossipServer).Recv-fm(0xc42037c398, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x4b
github.com/cockroachdb/cockroach/gossip.(*server).gossipReceiver(0xc42037c380, 0xc420026810, 0xc4206b6ac0, 0xc42142bf40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:310 +0x604
github.com/cockroachdb/cockroach/gossip.(*server).Gossip.func3.1()
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x116
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4206b6b00)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17258 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a8cae0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17848 [select]:
github.com/cockroachdb/cockroach/storage.(*bookie).start.func1()
/go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x4c8
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc4209ae960)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17234 [IO wait]:
net.runtime_pollWait(0x7f033355b0b0, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc421258060, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc421258060, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc421258000, 0xc421408000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc420026000, 0xc421408000, 0x8000, 0x8000, 0x0, 0x6, 0x2)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc420026000, 0xc421408000, 0x8000, 0x8000, 0x0, 0x7fd1, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc4219b2120)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc4219b2120, 0xc4219b01b8, 0x9, 0x9, 0x10, 0x10, 0xc420a7b1a0)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc4219b2120, 0xc4219b01b8, 0x9, 0x9, 0x9, 0x61a47a, 0xc42028bd40, 0xc420f3cff8)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc4219b2120, 0xc4219b01b8, 0x9, 0x9, 0x0, 0xc42116adc0, 0xc420034a20)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc4219b01b8, 0x9, 0x9, 0x283d340, 0xc4219b2120, 0x0, 0xc400000000, 0xc420f3cff8, 0x28411c0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc4219b0180, 0x0, 0x283d8c0, 0xc420010170, 0xc422265a20)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc4213ee0f0, 0xc420f3d0e0, 0xc420f3d0e0, 0xc420034ce0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc421a20000, 0xc4213ee1b0)
/go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x30a
google.golang.org/grpc.(*Server).serveStreams(0xc4214c6000, 0x2856e20, 0xc421a20000)
/go/src/google.golang.org/grpc/server.go:424 +0x1d6
google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc4214c6000, 0x28573c0, 0xc420026000, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:411 +0x4f1
google.golang.org/grpc.(*Server).handleRawConn(0xc4214c6000, 0x28573c0, 0xc420026000)
/go/src/google.golang.org/grpc/server.go:388 +0x5c3
created by google.golang.org/grpc.(*Server).Serve
/go/src/google.golang.org/grpc/server.go:357 +0x20a
goroutine 17551 [select]:
github.com/cockroachdb/cockroach/gossip.(*client).gossip(0xc421478c40, 0x7f0333502738, 0xc420e2a000, 0xc4205d9680, 0x2857180, 0xc4216bc0e0, 0xc420cc4090, 0xc4216bc080, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:299 +0x4b5
github.com/cockroachdb/cockroach/gossip.(*client).start.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:126 +0x72d
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc42146dfc0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17302 [IO wait]:
net.runtime_pollWait(0x7f033355b2f0, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc4210760d0, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc4210760d0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc421076070, 0xc42198c000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc420026490, 0xc42198c000, 0x8000, 0x8000, 0xc420ec33e8, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc420026490, 0xc42198c000, 0x8000, 0x8000, 0x0, 0x7fd4, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc4211ba840)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc4211ba840, 0xc42192db38, 0x9, 0x9, 0x0, 0x0, 0xc421b03b58)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc4211ba840, 0xc42192db38, 0x9, 0x9, 0x9, 0xc420ec2f68, 0xc420074340, 0x0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc4211ba840, 0xc42192db38, 0x9, 0x9, 0x1, 0xc421ff6130, 0x0)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc42192db38, 0x9, 0x9, 0x283d340, 0xc4211ba840, 0x0, 0x0, 0xc4221231d0, 0x0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc42192db00, 0x0, 0x0, 0x0, 0x0)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc4213b8e70, 0xc420ec33e0, 0xc420ec33e0, 0xc421b03ed0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Client).reader(0xc420ba6200)
/go/src/google.golang.org/grpc/transport/http2_client.go:921 +0x154
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:203 +0xc1a
goroutine 17272 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc4218d8420)
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x64
created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0xb61
goroutine 17889 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc42038c920)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17897 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421140960)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17280 [IO wait]:
net.runtime_pollWait(0x7f033355b5f0, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc421a913a0, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc421a913a0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc421a91340, 0xc420ffe000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc420026480, 0xc420ffe000, 0x8000, 0x8000, 0xc421803418, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc420026480, 0xc420ffe000, 0x8000, 0x8000, 0x0, 0x7fd4, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc4218d9c80)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc4218d9c80, 0xc42192d6b8, 0x9, 0x9, 0x0, 0x0, 0xc420d79b58)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc4218d9c80, 0xc42192d6b8, 0x9, 0x9, 0x9, 0xc420ec2608, 0xc420075d40, 0x0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc4218d9c80, 0xc42192d6b8, 0x9, 0x9, 0x1, 0xc421baef80, 0x0)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc42192d6b8, 0x9, 0x9, 0x283d340, 0xc4218d9c80, 0x0, 0x0, 0xc422122e10, 0x0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc42192d680, 0x0, 0x0, 0x0, 0x0)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc4213b83c0, 0xc421803410, 0xc421803410, 0xc420d79ed0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Client).reader(0xc420ba6100)
/go/src/google.golang.org/grpc/transport/http2_client.go:921 +0x154
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:203 +0xc1a
goroutine 17579 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420e2a0c0, 0xc421766e70, 0x5, 0x5, 0x0, 0x0, 0x0)
/go/src/google.golang.org/grpc/transport/transport.go:141 +0x89c
google.golang.org/grpc/transport.(*Stream).Read(0xc4219ceff0, 0xc421766e70, 0x5, 0x5, 0xc420249ae0, 0xc420249ae0, 0x96fe7f)
/go/src/google.golang.org/grpc/transport/transport.go:314 +0x79
io.ReadAtLeast(0x2841080, 0xc4219ceff0, 0xc421766e70, 0x5, 0x5, 0x5, 0x61a35f, 0xc420249c3c, 0x0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x2841080, 0xc4219ceff0, 0xc421766e70, 0x5, 0x5, 0xc420b52958, 0xc420249c3c, 0x5e6850)
/usr/local/go/src/io/io.go:325 +0x73
google.golang.org/grpc.(*parser).recvMsg(0xc421766e60, 0x7fffffff, 0xc420e2a300, 0xc420249c50, 0x638110, 0xc420e2a340, 0xc420249e80, 0xc36e73)
/go/src/google.golang.org/grpc/rpc_util.go:231 +0xa4
google.golang.org/grpc.recv(0xc421766e60, 0x284fac0, 0x354ff60, 0xc4219ceff0, 0x0, 0x0, 0x1bf28c0, 0xc42021eeb0, 0x7fffffff, 0x50, ...)
/go/src/google.golang.org/grpc/rpc_util.go:315 +0x5e
google.golang.org/grpc.(*clientStream).RecvMsg(0xc42132eb40, 0x1bf28c0, 0xc42021eeb0, 0x0, 0x0)
/go/src/google.golang.org/grpc/stream.go:305 +0x15e
github.com/cockroachdb/cockroach/gossip.(*gossipGossipClient).Recv(0xc4216bc0e0, 0xc4205d9680, 0xc4201e33b0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:194 +0x86
github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2.1(0x2857180, 0xc4216bc0e0, 0xc421478c40, 0xc4205d9680, 0xc4218b6748, 0xc67b3d)
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:287 +0x43
github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2()
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:295 +0xb7
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc420e2a240)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17304 [select, 6 minutes]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc421914640)
/go/src/google.golang.org/grpc/clientconn.go:720 +0x7d7
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
/go/src/google.golang.org/grpc/clientconn.go:467 +0x9c9
goroutine 17273 [select]:
github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1()
/go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0x12f6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc42059deb0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17366 [select, 6 minutes]:
google.golang.org/grpc/transport.(*http2Server).controller(0xc421a21680)
/go/src/google.golang.org/grpc/transport/http2_server.go:673 +0xa56
created by google.golang.org/grpc/transport.newHTTP2Server
/go/src/google.golang.org/grpc/transport/http2_server.go:139 +0xa91
goroutine 17230 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x86
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc42125fbc0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17907 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2745 +0x184a
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc4214ab0c0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17373 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc420a4e240)
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x64
created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0xb61
goroutine 17330 [IO wait]:
net.runtime_pollWait(0x7f0324e287a8, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc4210761b0, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc4210761b0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc421076150, 0xc421a92000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc42011a3e8, 0xc421a92000, 0x8000, 0x8000, 0x0, 0x6, 0x0)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc42011a3e8, 0xc421a92000, 0x8000, 0x8000, 0x0, 0x7fd1, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc42165b260)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc42165b260, 0xc4218ef3b8, 0x9, 0x9, 0x10, 0x10, 0xc421ff6080)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc42165b260, 0xc4218ef3b8, 0x9, 0x9, 0x9, 0x61a47a, 0xc4201c5380, 0xc420ec3088)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc42165b260, 0xc4218ef3b8, 0x9, 0x9, 0x0, 0xc420cd3810, 0xc420468a20)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc4218ef3b8, 0x9, 0x9, 0x283d340, 0xc42165b260, 0x0, 0xc400000000, 0xc420ec3088, 0x28411c0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc4218ef380, 0x0, 0x283d8c0, 0xc420010170, 0xc421c35000)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc421376ba0, 0xc420ec3170, 0xc420ec3170, 0xc420468ce0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420cc4a20, 0xc421376c30)
/go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x30a
google.golang.org/grpc.(*Server).serveStreams(0xc420474000, 0x2856e20, 0xc420cc4a20)
/go/src/google.golang.org/grpc/server.go:424 +0x1d6
google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc420474000, 0x28573c0, 0xc42011a3e8, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:411 +0x4f1
google.golang.org/grpc.(*Server).handleRawConn(0xc420474000, 0x28573c0, 0xc42011a3e8)
/go/src/google.golang.org/grpc/server.go:388 +0x5c3
created by google.golang.org/grpc.(*Server).Serve
/go/src/google.golang.org/grpc/server.go:357 +0x20a
goroutine 17307 [select, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x5a6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc420d30fa0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17873 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:348 +0x519
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc420dfe060)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17254 [chan receive]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc4219b32c0)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17231 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7f033355b170, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc420b4f480, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc420b4f480, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).accept(0xc420b4f420, 0x0, 0x2841f80, 0xc421a8c0c0)
/usr/local/go/src/net/fd_unix.go:419 +0x2b8
net.(*TCPListener).accept(0xc42011a240, 0xc420026000, 0xc420038e98, 0x6146ef)
/usr/local/go/src/net/tcpsock_posix.go:132 +0x51
net.(*TCPListener).Accept(0xc42011a240, 0x1f66120, 0xc4214c6000, 0x28573c0, 0xc420026000)
/usr/local/go/src/net/tcpsock.go:222 +0x50
google.golang.org/grpc.(*Server).Serve(0xc4214c6000, 0x284edc0, 0xc42011a240, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:348 +0x1b8
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x54
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421aeba20)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17431 [IO wait]:
net.runtime_pollWait(0x7f033355a9f0, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc4202dfc60, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc4202dfc60, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc4202dfc00, 0xc420e78000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc42011a608, 0xc420e78000, 0x8000, 0x8000, 0xc421a31208, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc42011a608, 0xc420e78000, 0x8000, 0x8000, 0x0, 0x7fd4, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc420746ba0)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc420746ba0, 0xc420cd5838, 0x9, 0x9, 0x0, 0x0, 0xc420e8fb58)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc420746ba0, 0xc420cd5838, 0x9, 0x9, 0x9, 0xc421a30d88, 0xc4202681a0, 0x0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc420746ba0, 0xc420cd5838, 0x9, 0x9, 0x1, 0xc4212ea890, 0x0)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc420cd5838, 0x9, 0x9, 0x283d340, 0xc420746ba0, 0x0, 0x0, 0xc421a2c870, 0x0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc420cd5800, 0x0, 0x0, 0x0, 0x0)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc4206a5fb0, 0xc421a31200, 0xc421a31200, 0xc420e8fed0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Client).reader(0xc420901700)
/go/src/google.golang.org/grpc/transport/http2_client.go:921 +0x154
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:203 +0xc1a
goroutine 17911 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:1023 +0x2f9
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc4214ab100)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17893 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc42038cac0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17288 [select]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x416
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4218fb8e0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17895 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421140960)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17892 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420af5000, 0xecf57acee, 0xc41dfaaec3, 0x2b92920, 0xc4207dc200, 0xc421870cf0, 0xc421790000, 0xc4214a64a0)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x299
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1.1(0xc421790000, 0xc4210ad680)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:259 +0x124
github.com/cockroachdb/cockroach/storage.(*storeRangeSet).Visit(0xc4210ad740, 0xc421952380)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:243 +0x453
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:261 +0x3a5
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc42038c9a0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17887 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc42038c8e0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17518 [IO wait]:
net.runtime_pollWait(0x7f033355b470, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc420986450, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc420986450, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc4209863f0, 0xc420fe2000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc42011a7f0, 0xc420fe2000, 0x8000, 0x8000, 0xc421f2cbd8, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc42011a7f0, 0xc420fe2000, 0x8000, 0x8000, 0x0, 0x7fd4, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc42075ad80)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc42075ad80, 0xc420b5af38, 0x9, 0x9, 0x0, 0x0, 0xc4214e1b58)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc42075ad80, 0xc420b5af38, 0x9, 0x9, 0x9, 0xc421c13778, 0xc420269040, 0x0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc42075ad80, 0xc420b5af38, 0x9, 0x9, 0x1, 0xc420602050, 0x0)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc420b5af38, 0x9, 0x9, 0x283d340, 0xc42075ad80, 0x0, 0x0, 0xc421c0ce10, 0x0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc420b5af00, 0x0, 0x0, 0x0, 0x0)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc420bf7350, 0xc421f2cbd0, 0xc421f2cbd0, 0xc4214e1ed0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Client).reader(0xc420901f00)
/go/src/google.golang.org/grpc/transport/http2_client.go:921 +0x154
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:203 +0xc1a
goroutine 17303 [select, 5 minutes]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc420ba6200)
/go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x8c9
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:233 +0x14bd
goroutine 17344 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*server).start.func3()
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x94
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421924700)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17885 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421140240)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17339 [select, 6 minutes]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421974900, 0xc4219244b0, 0x5, 0x5, 0x0, 0x0, 0x0)
/go/src/google.golang.org/grpc/transport/transport.go:141 +0x89c
google.golang.org/grpc/transport.(*Stream).Read(0xc4219ce5a0, 0xc4219244b0, 0x5, 0x5, 0xc42143db80, 0xc4205de2b0, 0xc4205de294)
/go/src/google.golang.org/grpc/transport/transport.go:314 +0x79
io.ReadAtLeast(0x2841080, 0xc4219ce5a0, 0xc4219244b0, 0x5, 0x5, 0x5, 0xc4212b17e8, 0x61a47a, 0xc420448ea0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x2841080, 0xc4219ce5a0, 0xc4219244b0, 0x5, 0x5, 0xc4ffffffff, 0x0, 0xc4205de240)
/usr/local/go/src/io/io.go:325 +0x73
google.golang.org/grpc.(*parser).recvMsg(0xc4219244a0, 0x7fffffff, 0x5e25a5, 0xc420b74898, 0x0, 0x0, 0x6, 0x0)
/go/src/google.golang.org/grpc/rpc_util.go:231 +0xa4
google.golang.org/grpc.recv(0xc4219244a0, 0x284fac0, 0x354ff60, 0xc4219ce5a0, 0x0, 0x0, 0x1c03280, 0xc42192b880, 0x7fffffff, 0x1, ...)
/go/src/google.golang.org/grpc/rpc_util.go:315 +0x5e
google.golang.org/grpc.(*serverStream).RecvMsg(0xc420cc4d80, 0x1c03280, 0xc42192b880, 0x0, 0x0)
/go/src/google.golang.org/grpc/stream.go:492 +0x154
github.com/cockroachdb/cockroach/gossip.(*gossipGossipServer).Recv(0xc42090f3e0, 0xc4ffffffff, 0x2, 0x200180002)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:230 +0x86
github.com/cockroachdb/cockroach/gossip.(Gossip_GossipServer).Recv-fm(0xc421478018, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x4b
github.com/cockroachdb/cockroach/gossip.(*server).gossipReceiver(0xc421478000, 0xc42011a468, 0xc421974a80, 0xc4212b1f40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:310 +0x604
github.com/cockroachdb/cockroach/gossip.(*server).Gossip.func3.1()
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x116
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421974ac0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17519 [select, 5 minutes]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc420901f00)
/go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x8c9
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:233 +0x14bd
goroutine 17923 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc421796880, 0xecf57acee, 0xc41f4160aa, 0x2b92920, 0xc4207dc200, 0xc421870ea0, 0x0, 0xc420f69f58)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x299
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x540
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc42038cfa0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17429 [IO wait]:
net.runtime_pollWait(0x7f033355aab0, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc42073a300, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc42073a300, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc42073a2a0, 0xc420fba000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc42011a600, 0xc420fba000, 0x8000, 0x8000, 0x0, 0x6, 0x0)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc42011a600, 0xc420fba000, 0x8000, 0x8000, 0x0, 0x7fca, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc420746900)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc420746900, 0xc420cd5778, 0x9, 0x9, 0x10, 0x10, 0xc4212ea7e0)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc420746900, 0xc420cd5778, 0x9, 0x9, 0x9, 0x61a47a, 0xc420449860, 0xc421a30ea8)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc420746900, 0xc420cd5778, 0x9, 0x9, 0x0, 0xc4207071d0, 0xc420fa7a20)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc420cd5778, 0x9, 0x9, 0x283d340, 0xc420746900, 0x0, 0xc400000000, 0xc421a30ea8, 0x28411c0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc420cd5740, 0x0, 0x283d8c0, 0xc420010170, 0xc421a36de0)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc4206a5d70, 0xc421a30f90, 0xc421a30f90, 0xc420fa7ce0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420cc5c20, 0xc4206a5e90)
/go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x30a
google.golang.org/grpc.(*Server).serveStreams(0xc4211ee640, 0x2856e20, 0xc420cc5c20)
/go/src/google.golang.org/grpc/server.go:424 +0x1d6
google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc4211ee640, 0x28573c0, 0xc42011a600, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:411 +0x4f1
google.golang.org/grpc.(*Server).handleRawConn(0xc4211ee640, 0x28573c0, 0xc42011a600)
/go/src/google.golang.org/grpc/server.go:388 +0x5c3
created by google.golang.org/grpc.(*Server).Serve
/go/src/google.golang.org/grpc/server.go:357 +0x20a
goroutine 17847 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:348 +0x519
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc4209ae6e0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17841 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42038c3c0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17326 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x86
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4213763c0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17522 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*server).start.func3()
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x94
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4218e5b80)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17910 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:1003 +0x2f9
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc4214ab0f0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17308 [select]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x416
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc420d30fb0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17350 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421a1fd40, 0xc421a0fd90, 0x5, 0x5, 0x0, 0x0, 0x0)
/go/src/google.golang.org/grpc/transport/transport.go:141 +0x89c
google.golang.org/grpc/transport.(*Stream).Read(0xc4206ad590, 0xc421a0fd90, 0x5, 0x5, 0xc4212abae0, 0xc4212abae0, 0x96fe7f)
/go/src/google.golang.org/grpc/transport/transport.go:314 +0x79
io.ReadAtLeast(0x2841080, 0xc4206ad590, 0xc421a0fd90, 0x5, 0x5, 0x5, 0x61a35f, 0xc4212abc3c, 0x0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x2841080, 0xc4206ad590, 0xc421a0fd90, 0x5, 0x5, 0xc4206a5678, 0xc4212abc3c, 0x5e6850)
/usr/local/go/src/io/io.go:325 +0x73
google.golang.org/grpc.(*parser).recvMsg(0xc421a0fd80, 0x7fffffff, 0xc421a1f780, 0xc4212abc50, 0x638110, 0xc421a1f800, 0xc4212abe80, 0xc36e73)
/go/src/google.golang.org/grpc/rpc_util.go:231 +0xa4
google.golang.org/grpc.recv(0xc421a0fd80, 0x284fac0, 0x354ff60, 0xc4206ad590, 0x0, 0x0, 0x1bf28c0, 0xc4201e3720, 0x7fffffff, 0x50, ...)
/go/src/google.golang.org/grpc/rpc_util.go:315 +0x5e
google.golang.org/grpc.(*clientStream).RecvMsg(0xc421966c60, 0x1bf28c0, 0xc4201e3720, 0x0, 0x0)
/go/src/google.golang.org/grpc/stream.go:305 +0x15e
github.com/cockroachdb/cockroach/gossip.(*gossipGossipClient).Recv(0xc420d31420, 0xc42081a900, 0xc4208dacd0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:194 +0x86
github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2.1(0x2857180, 0xc420d31420, 0xc42037c460, 0xc42081a900, 0xc420931748, 0xc67b3d)
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:287 +0x43
github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2()
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:295 +0xb7
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421a1fe00)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17524 [select]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x416
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421539630)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17901 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc42038ce60)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17891 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc42038c960)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17906 [select]:
github.com/cockroachdb/cockroach/storage.(*bookie).start.func1()
/go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x4c8
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc420dfe2e0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17253 [chan receive]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc4219b32c0)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17883 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421140240)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17474 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7f033355ac30, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc420986220, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc420986220, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).accept(0xc4209861c0, 0x0, 0x2841f80, 0xc4218a4140)
/usr/local/go/src/net/fd_unix.go:419 +0x2b8
net.(*TCPListener).accept(0xc420026730, 0xc420026750, 0xc4218c9e98, 0x6146ef)
/usr/local/go/src/net/tcpsock_posix.go:132 +0x51
net.(*TCPListener).Accept(0xc420026730, 0x1f66120, 0xc421187040, 0x28573c0, 0xc420026750)
/usr/local/go/src/net/tcpsock.go:222 +0x50
google.golang.org/grpc.(*Server).Serve(0xc421187040, 0x284edc0, 0xc420026730, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:348 +0x1b8
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x54
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4218d57c0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17498 [select]:
github.com/cockroachdb/cockroach/gossip.(*client).gossip(0xc42037c9a0, 0x7f0333502738, 0xc421033440, 0xc42081b200, 0x2857180, 0xc421179170, 0xc420cc4090, 0xc421179110, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:299 +0x4b5
github.com/cockroachdb/cockroach/gossip.(*client).start.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:126 +0x72d
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421033400)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17281 [select, 3 minutes]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc420ba6100)
/go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x8c9
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:233 +0x14bd
goroutine 17239 [select]:
github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc421aa8000, 0xc4218fdb00, 0xc4218fb470, 0xf, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:282 +0x813
github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1()
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:205 +0xb3
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4213ee3c0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17266 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc421aa9600, 0xecf57aced, 0xc4031273f4, 0x2b92920, 0xc4207dc200, 0xc420cc41b0, 0xc420656c00, 0xc421973b20)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x299
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1.1(0xc420656c00, 0xc42125f3b0)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:259 +0x124
github.com/cockroachdb/cockroach/storage.(*storeRangeSet).Visit(0xc42125f860, 0xc421ae1500)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:243 +0x453
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:261 +0x3a5
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a8cc20)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17407 [select]:
github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1()
/go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0x12f6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc420a262d0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17242 [select]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x416
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc42059c0e0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17405 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421908500, 0xc421925a30, 0x5, 0x5, 0x0, 0x0, 0x0)
/go/src/google.golang.org/grpc/transport/transport.go:141 +0x89c
google.golang.org/grpc/transport.(*Stream).Read(0xc4219ce960, 0xc421925a30, 0x5, 0x5, 0xc420fadae0, 0xc420fadae0, 0x96fe7f)
/go/src/google.golang.org/grpc/transport/transport.go:314 +0x79
io.ReadAtLeast(0x2841080, 0xc4219ce960, 0xc421925a30, 0x5, 0x5, 0x5, 0x61a35f, 0xc420fadc3c, 0x0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x2841080, 0xc4219ce960, 0xc421925a30, 0x5, 0x5, 0x60db5b, 0xc420fadb68, 0x638220)
/usr/local/go/src/io/io.go:325 +0x73
google.golang.org/grpc.(*parser).recvMsg(0xc421925a20, 0x7fffffff, 0xc421909040, 0xc420fadc50, 0x638110, 0xc421909080, 0xc420fade80, 0xc36e73)
/go/src/google.golang.org/grpc/rpc_util.go:231 +0xa4
google.golang.org/grpc.recv(0xc421925a20, 0x284fac0, 0x354ff60, 0xc4219ce960, 0x0, 0x0, 0x1bf28c0, 0xc42021ef00, 0x7fffffff, 0x50, ...)
/go/src/google.golang.org/grpc/rpc_util.go:315 +0x5e
google.golang.org/grpc.(*clientStream).RecvMsg(0xc420ca0fc0, 0x1bf28c0, 0xc42021ef00, 0x0, 0x0)
/go/src/google.golang.org/grpc/stream.go:305 +0x15e
github.com/cockroachdb/cockroach/gossip.(*gossipGossipClient).Recv(0xc420a26120, 0xc4205d8480, 0xc42059bdb0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:194 +0x86
github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2.1(0x2857180, 0xc420a26120, 0xc421478700, 0xc4205d8480, 0xc420f67748, 0xc67b3d)
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:287 +0x43
github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2()
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:295 +0xb7
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421908580)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17890 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc42038c940)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17263 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a8cb80)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17552 [select, 6 minutes]:
google.golang.org/grpc.NewClientStream.func2(0x2856dc0, 0xc420ba6100, 0xc4219ceff0, 0xc42132eb40)
/go/src/google.golang.org/grpc/stream.go:197 +0x54e
created by google.golang.org/grpc.NewClientStream
/go/src/google.golang.org/grpc/stream.go:217 +0x10a5
goroutine 17881 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc42190d800, 0xecf57acee, 0xc41d82f65a, 0x2b92920, 0xc4207dc200, 0xc421870b40, 0xc4218f3b00, 0xc4212673a0)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x299
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1.1(0xc4218f3b00, 0xc4210ac330)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:259 +0x124
github.com/cockroachdb/cockroach/storage.(*storeRangeSet).Visit(0xc4210ac3f0, 0xc421952180)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:243 +0x453
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:261 +0x3a5
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42038c5a0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17432 [select, 5 minutes]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc420901700)
/go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x8c9
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:233 +0x14bd
goroutine 17922 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc42038cf80)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17868 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:1023 +0x2f9
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc4214aa540)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17358 [select]:
github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1()
/go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0x12f6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc420d318b0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17414 [select]:
github.com/cockroachdb/cockroach/gossip.(*server).Gossip(0xc421478000, 0x2856880, 0xc420a27160, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:180 +0x73e
github.com/cockroachdb/cockroach/gossip._Gossip_Gossip_Handler(0x1be3a80, 0xc421478000, 0x2852e00, 0xc420cc5950, 0x33, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:211 +0xda
google.golang.org/grpc.(*Server).processStreamingRPC(0xc4214c6000, 0x2856e20, 0xc421a20000, 0xc4219ceb40, 0xc42125ed50, 0x28263a0, 0xc4206a4ff0, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:689 +0xc7d
google.golang.org/grpc.(*Server).handleStream(0xc4214c6000, 0x2856e20, 0xc421a20000, 0xc4219ceb40, 0xc4206a4ff0)
/go/src/google.golang.org/grpc/server.go:773 +0x170e
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42059c040, 0xc4214c6000, 0x2856e20, 0xc421a20000, 0xc4219ceb40)
/go/src/google.golang.org/grpc/server.go:422 +0xb9
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/google.golang.org/grpc/server.go:423 +0xbb
goroutine 17396 [select]:
github.com/cockroachdb/cockroach/gossip.(*client).gossip(0xc421478700, 0x7f0333502738, 0xc421908440, 0xc4205d8480, 0x2857180, 0xc420a26120, 0xc420cc4090, 0xc420a260c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:299 +0x4b5
github.com/cockroachdb/cockroach/gossip.(*client).start.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:126 +0x72d
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4219082c0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17904 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc42038cec0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17369 [select, 6 minutes]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4219095c0, 0xc421906d90, 0x5, 0x5, 0x0, 0x0, 0x0)
/go/src/google.golang.org/grpc/transport/transport.go:141 +0x89c
google.golang.org/grpc/transport.(*Stream).Read(0xc4219ceb40, 0xc421906d90, 0x5, 0x5, 0xc4203c8c80, 0xc42066eeb0, 0xc42066ee94)
/go/src/google.golang.org/grpc/transport/transport.go:314 +0x79
io.ReadAtLeast(0x2841080, 0xc4219ceb40, 0xc421906d90, 0x5, 0x5, 0x5, 0xc4214e77e8, 0x61a47a, 0xc420265380)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x2841080, 0xc4219ceb40, 0xc421906d90, 0x5, 0x5, 0xc4ffffffff, 0x0, 0xc42066ee40)
/usr/local/go/src/io/io.go:325 +0x73
google.golang.org/grpc.(*parser).recvMsg(0xc421906d80, 0x7fffffff, 0x5e25a5, 0xc4209b9918, 0x0, 0x0, 0x6, 0x0)
/go/src/google.golang.org/grpc/rpc_util.go:231 +0xa4
google.golang.org/grpc.recv(0xc421906d80, 0x284fac0, 0x354ff60, 0xc4219ceb40, 0x0, 0x0, 0x1c03280, 0xc42192b480, 0x7fffffff, 0x1, ...)
/go/src/google.golang.org/grpc/rpc_util.go:315 +0x5e
google.golang.org/grpc.(*serverStream).RecvMsg(0xc420cc5950, 0x1c03280, 0xc42192b480, 0x0, 0x0)
/go/src/google.golang.org/grpc/stream.go:492 +0x154
github.com/cockroachdb/cockroach/gossip.(*gossipGossipServer).Recv(0xc420a27160, 0xc4ffffffff, 0x6, 0x600180002)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:230 +0x86
github.com/cockroachdb/cockroach/gossip.(Gossip_GossipServer).Recv-fm(0xc421478018, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x4b
github.com/cockroachdb/cockroach/gossip.(*server).gossipReceiver(0xc421478000, 0xc42011a5c8, 0xc421909740, 0xc4214e7f40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:310 +0x604
github.com/cockroachdb/cockroach/gossip.(*server).Gossip.func3.1()
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x116
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc42192b100)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17286 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*server).start.func3()
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x94
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4219be080)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17479 [select, 6 minutes]:
google.golang.org/grpc/transport.(*http2Server).controller(0xc421a21ef0)
/go/src/google.golang.org/grpc/transport/http2_server.go:673 +0xa56
created by google.golang.org/grpc/transport.newHTTP2Server
/go/src/google.golang.org/grpc/transport/http2_server.go:139 +0xa91
goroutine 17866 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).Start.func3()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:927 +0x19d
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc4209afcc0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17340 [IO wait]:
net.runtime_pollWait(0x7f033355b770, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc421268060, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc421268060, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc421268000, 0xc42108c000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc42011a470, 0xc42108c000, 0x8000, 0x8000, 0xc420ec2de8, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc42011a470, 0xc42108c000, 0x8000, 0x8000, 0x0, 0x7fd4, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc420b75200)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc420b75200, 0xc4218efe38, 0x9, 0x9, 0x0, 0x0, 0xc4210c5b58)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc420b75200, 0xc4218efe38, 0x9, 0x9, 0x9, 0xc420ec2698, 0xc4205c0820, 0x0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc420b75200, 0xc4218efe38, 0x9, 0x9, 0x1, 0xc420efdf90, 0x0)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc4218efe38, 0x9, 0x9, 0x283d340, 0xc420b75200, 0x0, 0x0, 0xc422122f00, 0x0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc4218efe00, 0x0, 0x0, 0x0, 0x0)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc420ea41b0, 0xc420ec2de0, 0xc420ec2de0, 0xc4210c5ed0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Client).reader(0xc420409600)
/go/src/google.golang.org/grpc/transport/http2_client.go:921 +0x154
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:203 +0xc1a
goroutine 17898 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421140960)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17323 [select]:
github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1()
/go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0x12f6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc42090e050)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17225 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/rpc.NewContext.func1()
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:122 +0x95
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421aea1a0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17240 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*server).start.func3()
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x94
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421a8c200)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17237 [select]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc421468200)
/go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x8c9
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:233 +0x14bd
goroutine 17285 [select]:
github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc421aa8000, 0xc421469f00, 0xc420d30af0, 0xf, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:282 +0x813
github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1()
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:205 +0xb3
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4213cf1d0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17256 [chan receive]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc4219b32c0)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17909 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).Start.func3()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:927 +0x19d
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc420dfe5a0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17517 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421033500, 0xc4218a4ef0, 0x5, 0x5, 0x0, 0x0, 0x0)
/go/src/google.golang.org/grpc/transport/transport.go:141 +0x89c
google.golang.org/grpc/transport.(*Stream).Read(0xc4206adc20, 0xc4218a4ef0, 0x5, 0x5, 0xc420245ae0, 0xc420245ae0, 0x96fe7f)
/go/src/google.golang.org/grpc/transport/transport.go:314 +0x79
io.ReadAtLeast(0x2841080, 0xc4206adc20, 0xc4218a4ef0, 0x5, 0x5, 0x5, 0x61a35f, 0xc420245c3c, 0x0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x2841080, 0xc4206adc20, 0xc4218a4ef0, 0x5, 0x5, 0xc420c1cdd8, 0xc420245c3c, 0x5e6850)
/usr/local/go/src/io/io.go:325 +0x73
google.golang.org/grpc.(*parser).recvMsg(0xc4218a4ee0, 0x7fffffff, 0xc420df1a80, 0xc420245c50, 0x638110, 0xc420df1b00, 0xc420245e80, 0xc36e73)
/go/src/google.golang.org/grpc/rpc_util.go:231 +0xa4
google.golang.org/grpc.recv(0xc4218a4ee0, 0x284fac0, 0x354ff60, 0xc4206adc20, 0x0, 0x0, 0x1bf28c0, 0xc42021ee60, 0x7fffffff, 0x50, ...)
/go/src/google.golang.org/grpc/rpc_util.go:315 +0x5e
google.golang.org/grpc.(*clientStream).RecvMsg(0xc42015b440, 0x1bf28c0, 0xc42021ee60, 0x0, 0x0)
/go/src/google.golang.org/grpc/stream.go:305 +0x15e
github.com/cockroachdb/cockroach/gossip.(*gossipGossipClient).Recv(0xc421179170, 0xc42081b200, 0xc4201e3630, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:194 +0x86
github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2.1(0x2857180, 0xc421179170, 0xc42037c9a0, 0xc42081b200, 0xc4218b3748, 0xc67b3d)
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:287 +0x43
github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2()
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:295 +0xb7
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc42146ccc0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17238 [select, 6 minutes]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc421b15680)
/go/src/google.golang.org/grpc/clientconn.go:720 +0x7d7
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
/go/src/google.golang.org/grpc/clientconn.go:467 +0x9c9
goroutine 17248 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:1023 +0x2f9
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc42059c6e0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17896 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421140960)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17277 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7f033355aff0, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc421a91170, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc421a91170, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).accept(0xc421a91110, 0x0, 0x2841f80, 0xc421aebf80)
/usr/local/go/src/net/fd_unix.go:419 +0x2b8
net.(*TCPListener).accept(0xc420026478, 0xc42011a248, 0xc420033e98, 0x6146ef)
/usr/local/go/src/net/tcpsock_posix.go:132 +0x51
net.(*TCPListener).Accept(0xc420026478, 0x1f66120, 0xc4213845a0, 0x28573c0, 0xc42011a248)
/usr/local/go/src/net/tcpsock.go:222 +0x50
google.golang.org/grpc.(*Server).Serve(0xc4213845a0, 0x284edc0, 0xc420026478, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:348 +0x1b8
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x54
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421a0ed40)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17854 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42037fd40)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17853 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42037fd40)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17501 [select]:
github.com/cockroachdb/cockroach/gossip.(*server).Gossip(0xc42037c380, 0x2856880, 0xc421179480, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:180 +0x73e
github.com/cockroachdb/cockroach/gossip._Gossip_Gossip_Handler(0x1be3a80, 0xc42037c380, 0x2852e00, 0xc420388090, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:211 +0xda
google.golang.org/grpc.(*Server).processStreamingRPC(0xc4213845a0, 0x2856e20, 0xc420cc4480, 0xc4206add10, 0xc4213eaf60, 0x28263a0, 0xc420bc30e0, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:689 +0xc7d
google.golang.org/grpc.(*Server).handleStream(0xc4213845a0, 0x2856e20, 0xc420cc4480, 0xc4206add10, 0xc420bc30e0)
/go/src/google.golang.org/grpc/server.go:773 +0x170e
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4218fb850, 0xc4213845a0, 0x2856e20, 0xc420cc4480, 0xc4206add10)
/go/src/google.golang.org/grpc/server.go:422 +0xb9
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/google.golang.org/grpc/server.go:423 +0xbb
goroutine 17298 [select, 6 minutes]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc4219a9cc0)
/go/src/google.golang.org/grpc/clientconn.go:720 +0x7d7
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
/go/src/google.golang.org/grpc/clientconn.go:467 +0x9c9
goroutine 17342 [select, 6 minutes]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc4219c7040)
/go/src/google.golang.org/grpc/clientconn.go:720 +0x7d7
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
/go/src/google.golang.org/grpc/clientconn.go:467 +0x9c9
goroutine 17365 [IO wait]:
net.runtime_pollWait(0x7f033355adb0, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc4212680d0, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc4212680d0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc421268070, 0xc4213e0000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc420026590, 0xc4213e0000, 0x8000, 0x8000, 0x0, 0x6, 0x0)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc420026590, 0xc4213e0000, 0x8000, 0x8000, 0x0, 0x7fd1, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc420b835c0)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc420b835c0, 0xc4210fcb78, 0x9, 0x9, 0x10, 0x10, 0xc420efde90)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc420b835c0, 0xc4210fcb78, 0x9, 0x9, 0x9, 0x61a47a, 0xc42062dba0, 0xc420ec2a88)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc420b835c0, 0xc4210fcb78, 0x9, 0x9, 0x0, 0xc420577860, 0xc4212ada10)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc4210fcb78, 0x9, 0x9, 0x283d340, 0xc420b835c0, 0x0, 0xc400000000, 0xc420ec2a88, 0x28411c0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc4210fcb40, 0x0, 0x283d8c0, 0xc420010170, 0xc421c34d20)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc4213399b0, 0xc420ec2b70, 0xc420ec2b70, 0xc4212adcd0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc421a21680, 0xc421339b30)
/go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x30a
google.golang.org/grpc.(*Server).serveStreams(0xc421385d60, 0x2856e20, 0xc421a21680)
/go/src/google.golang.org/grpc/server.go:424 +0x1d6
google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc421385d60, 0x28573c0, 0xc420026590, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:411 +0x4f1
google.golang.org/grpc.(*Server).handleRawConn(0xc421385d60, 0x28573c0, 0xc420026590)
/go/src/google.golang.org/grpc/server.go:388 +0x5c3
created by google.golang.org/grpc.(*Server).Serve
/go/src/google.golang.org/grpc/server.go:357 +0x20a
goroutine 17227 [select]:
github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1()
/go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0x12f6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4218fa1c0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17247 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:1003 +0x2f9
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc42059c6d0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17243 [select]:
github.com/cockroachdb/cockroach/storage.(*bookie).start.func1()
/go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x4c8
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a8c220)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17846 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc4204b2a50)
/go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xde7
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm()
/go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x42
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42012ec90)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17338 [select]:
github.com/cockroachdb/cockroach/gossip.(*server).Gossip(0xc421478000, 0x2856880, 0xc42090f3e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:180 +0x73e
github.com/cockroachdb/cockroach/gossip._Gossip_Gossip_Handler(0x1be3a80, 0xc421478000, 0x2852e00, 0xc420cc4d80, 0x33, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:211 +0xda
google.golang.org/grpc.(*Server).processStreamingRPC(0xc4214c6000, 0x2856e20, 0xc421a20000, 0xc4219ce5a0, 0xc42125ed50, 0x28263a0, 0xc421377e30, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:689 +0xc7d
google.golang.org/grpc.(*Server).handleStream(0xc4214c6000, 0x2856e20, 0xc421a20000, 0xc4219ce5a0, 0xc421377e30)
/go/src/google.golang.org/grpc/server.go:773 +0x170e
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42059c040, 0xc4214c6000, 0x2856e20, 0xc421a20000, 0xc4219ce5a0)
/go/src/google.golang.org/grpc/server.go:422 +0xb9
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/google.golang.org/grpc/server.go:423 +0xbb
goroutine 17244 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2745 +0x184a
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc42059c690)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17861 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc4204b2d10)
/go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xde7
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm()
/go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x42
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc42012fce0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17437 [select]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x416
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc420a27770)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17292 [select]:
github.com/cockroachdb/cockroach/gossip.(*client).gossip(0xc4214782a0, 0x7f0333502738, 0xc4219f6d00, 0xc4207ba600, 0x2857180, 0xc4218fbeb0, 0xc420cc4090, 0xc4218fbe50, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:299 +0x4b5
github.com/cockroachdb/cockroach/gossip.(*client).start.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:126 +0x72d
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4219f6940)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17877 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42038c500)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17306 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*server).start.func3()
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x94
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421a0f3c0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17884 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421140240)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17867 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:1003 +0x2f9
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc4214aa530)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17260 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a8cb20)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17287 [select, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x5a6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4218fb8d0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17863 [select]:
github.com/cockroachdb/cockroach/storage.(*bookie).start.func1()
/go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x4c8
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc4209af6c0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17562 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*idAllocator).start.func1()
/go/src/github.com/cockroachdb/cockroach/storage/id_alloc.go:124 +0xacc
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a22b10)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17259 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a8cb00)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17851 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42037fd40)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17886 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc42038c8c0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17857 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:1023 +0x2f9
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42012fae0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17880 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42038c560)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17283 [IO wait]:
net.runtime_pollWait(0x7f033355b830, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc420b4fdb0, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc420b4fdb0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc420b4fd50, 0xc42142e000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc42011a248, 0xc42142e000, 0x8000, 0x8000, 0x0, 0x6, 0x2)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc42011a248, 0xc42142e000, 0x8000, 0x8000, 0x0, 0x7fd1, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc421666420)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc421666420, 0xc420c11478, 0x9, 0x9, 0x10, 0x10, 0xc420efddf0)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc421666420, 0xc420c11478, 0x9, 0x9, 0x9, 0x61a47a, 0xc420bd8680, 0xc420ec27b8)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc421666420, 0xc420c11478, 0x9, 0x9, 0x0, 0xc420a511d0, 0xc420cada10)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc420c11478, 0x9, 0x9, 0x283d340, 0xc421666420, 0x0, 0xc400000000, 0xc420ec27b8, 0x28411c0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc420c11440, 0x0, 0x283d8c0, 0xc420010170, 0xc421c34c00)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc4213cefc0, 0xc420ec28a0, 0xc420ec28a0, 0xc420cadcd0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420cc4480, 0xc4213cf0e0)
/go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x30a
google.golang.org/grpc.(*Server).serveStreams(0xc4213845a0, 0x2856e20, 0xc420cc4480)
/go/src/google.golang.org/grpc/server.go:424 +0x1d6
google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc4213845a0, 0x28573c0, 0xc42011a248, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:411 +0x4f1
google.golang.org/grpc.(*Server).handleRawConn(0xc4213845a0, 0x28573c0, 0xc42011a248)
/go/src/google.golang.org/grpc/server.go:388 +0x5c3
created by google.golang.org/grpc.(*Server).Serve
/go/src/google.golang.org/grpc/server.go:357 +0x20a
goroutine 17322 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc421667560)
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x64
created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0xb61
goroutine 17900 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc42038ce40)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17521 [select]:
github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc421aa8000, 0xc42043ea00, 0xc4211786b0, 0xf, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:282 +0x813
github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1()
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:205 +0xb3
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc420bf74a0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17523 [select, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x5a6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421539620)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17264 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc421aa9580, 0xecf57aced, 0xc402f734fc, 0x2b92920, 0xc4207dc200, 0xc420cc41b0, 0xc420656c00, 0xc421973b20)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x299
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1.1(0xc420656c00, 0xc42125f3b0)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:259 +0x124
github.com/cockroachdb/cockroach/storage.(*storeRangeSet).Visit(0xc42125f470, 0xc421ae0a80)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:243 +0x453
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:261 +0x3a5
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a8cba0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17331 [select, 6 minutes]:
google.golang.org/grpc/transport.(*http2Server).controller(0xc420cc4a20)
/go/src/google.golang.org/grpc/transport/http2_server.go:673 +0xa56
created by google.golang.org/grpc/transport.newHTTP2Server
/go/src/google.golang.org/grpc/transport/http2_server.go:139 +0xa91
goroutine 17374 [select]:
github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1()
/go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0x12f6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc420ca5a70)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17872 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc4204b2fd0)
/go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xde7
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm()
/go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x42
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc4214aa740)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17433 [select, 6 minutes]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc42188c280)
/go/src/google.golang.org/grpc/clientconn.go:720 +0x7d7
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
/go/src/google.golang.org/grpc/clientconn.go:467 +0x9c9
goroutine 17229 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc4214f8000)
/go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xde7
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm()
/go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x42
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc4218fa800)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17235 [select, 6 minutes]:
google.golang.org/grpc/transport.(*http2Server).controller(0xc421a20000)
/go/src/google.golang.org/grpc/transport/http2_server.go:673 +0xa56
created by google.golang.org/grpc/transport.newHTTP2Server
/go/src/google.golang.org/grpc/transport/http2_server.go:139 +0xa91
goroutine 17345 [select, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x5a6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc42090f5a0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17357 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc4206defc0)
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x64
created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0xb61
goroutine 17903 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc42038cea0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 383979 [runnable]:
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask.func1(0xc420cc41b0, 0x253ed95, 0x16, 0x1f2, 0xc4217f8620)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:231
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:235 +0xca
goroutine 17855 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).Start.func3()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:927 +0x19d
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc4209af0a0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17856 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:1003 +0x2f9
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42012fad0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17436 [select, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x5a6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc420a27760)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17875 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42038c4a0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17265 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a8cc00)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17327 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7f033355b530, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc421a1be90, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc421a1be90, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).accept(0xc421a1be30, 0x0, 0x2841f80, 0xc4219bfd60)
/usr/local/go/src/net/fd_unix.go:419 +0x2b8
net.(*TCPListener).accept(0xc42011a3d0, 0xc42011a3e8, 0xc420467e98, 0x6146ef)
/usr/local/go/src/net/tcpsock_posix.go:132 +0x51
net.(*TCPListener).Accept(0xc42011a3d0, 0x1f66120, 0xc420474000, 0x28573c0, 0xc42011a3e8)
/usr/local/go/src/net/tcpsock.go:222 +0x50
google.golang.org/grpc.(*Server).Serve(0xc420474000, 0x284edc0, 0xc42011a3d0, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:348 +0x1b8
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x54
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4219bfbe0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17299 [select]:
github.com/cockroachdb/cockroach/gossip.(*server).Gossip(0xc421478000, 0x2856880, 0xc420d30c80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:180 +0x73e
github.com/cockroachdb/cockroach/gossip._Gossip_Gossip_Handler(0x1be3a80, 0xc421478000, 0x2852e00, 0xc421a20f30, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:211 +0xda
google.golang.org/grpc.(*Server).processStreamingRPC(0xc4214c6000, 0x2856e20, 0xc421a20000, 0xc4206ad2c0, 0xc42125ed50, 0x28263a0, 0xc4213b87e0, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:689 +0xc7d
google.golang.org/grpc.(*Server).handleStream(0xc4214c6000, 0x2856e20, 0xc421a20000, 0xc4206ad2c0, 0xc4213b87e0)
/go/src/google.golang.org/grpc/server.go:773 +0x170e
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42059c040, 0xc4214c6000, 0x2856e20, 0xc421a20000, 0xc4206ad2c0)
/go/src/google.golang.org/grpc/server.go:422 +0xb9
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/google.golang.org/grpc/server.go:423 +0xbb
goroutine 17241 [select, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:920 +0x5a6
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc42059c0d0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17410 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x86
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4206ebf80)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17434 [select]:
github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc421aa8000, 0xc420901200, 0xc420a26f10, 0xf, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:282 +0x813
github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1()
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:205 +0xb3
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421148300)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17499 [select, 6 minutes]:
google.golang.org/grpc.NewClientStream.func2(0x2856dc0, 0xc420ba6100, 0xc4206adc20, 0xc42015b440)
/go/src/google.golang.org/grpc/stream.go:197 +0x54e
created by google.golang.org/grpc.NewClientStream
/go/src/google.golang.org/grpc/stream.go:217 +0x10a5
goroutine 17411 [IO wait, 6 minutes]:
net.runtime_pollWait(0x7f033355b3b0, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc4202df8e0, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc4202df8e0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).accept(0xc4202df880, 0x0, 0x2841f80, 0xc4219076c0)
/usr/local/go/src/net/fd_unix.go:419 +0x2b8
net.(*TCPListener).accept(0xc42011a5c0, 0xc42011a600, 0xc420fa6e98, 0x6146ef)
/usr/local/go/src/net/tcpsock_posix.go:132 +0x51
net.(*TCPListener).Accept(0xc42011a5c0, 0x1f66120, 0xc4211ee640, 0x28573c0, 0xc42011a600)
/usr/local/go/src/net/tcpsock.go:222 +0x50
google.golang.org/grpc.(*Server).Serve(0xc4211ee640, 0x284edc0, 0xc42011a5c0, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:348 +0x1b8
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func2()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:58 +0x54
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421906a20)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17319 [select, 6 minutes]:
google.golang.org/grpc.NewClientStream.func2(0x2856dc0, 0xc421468200, 0xc4219ce0f0, 0xc421a28c60)
/go/src/google.golang.org/grpc/stream.go:197 +0x54e
created by google.golang.org/grpc.NewClientStream
/go/src/google.golang.org/grpc/stream.go:217 +0x10a5
goroutine 17862 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1()
/go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:348 +0x519
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc4209af480)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17262 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a8cb60)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17321 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4219f6e80, 0xc4219be8f0, 0x5, 0x5, 0x0, 0x0, 0x0)
/go/src/google.golang.org/grpc/transport/transport.go:141 +0x89c
google.golang.org/grpc/transport.(*Stream).Read(0xc4219ce0f0, 0xc4219be8f0, 0x5, 0x5, 0xc421427ae0, 0xc421427ae0, 0x96fe7f)
/go/src/google.golang.org/grpc/transport/transport.go:314 +0x79
io.ReadAtLeast(0x2841080, 0xc4219ce0f0, 0xc4219be8f0, 0x5, 0x5, 0x5, 0x61a35f, 0xc421427c3c, 0x0)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x2841080, 0xc4219ce0f0, 0xc4219be8f0, 0x5, 0x5, 0xc420ea5288, 0xc421427c3c, 0x5e6850)
/usr/local/go/src/io/io.go:325 +0x73
google.golang.org/grpc.(*parser).recvMsg(0xc4219be8e0, 0x7fffffff, 0xc4219f6f80, 0xc421427c50, 0x638110, 0xc4219f7000, 0xc421427e80, 0xc36e73)
/go/src/google.golang.org/grpc/rpc_util.go:231 +0xa4
google.golang.org/grpc.recv(0xc4219be8e0, 0x284fac0, 0x354ff60, 0xc4219ce0f0, 0x0, 0x0, 0x1bf28c0, 0xc4208db400, 0x7fffffff, 0x50, ...)
/go/src/google.golang.org/grpc/rpc_util.go:315 +0x5e
google.golang.org/grpc.(*clientStream).RecvMsg(0xc421a28c60, 0x1bf28c0, 0xc4208db400, 0x0, 0x0)
/go/src/google.golang.org/grpc/stream.go:305 +0x15e
github.com/cockroachdb/cockroach/gossip.(*gossipGossipClient).Recv(0xc4218fbeb0, 0xc4207ba600, 0xc4208da6e0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:194 +0x86
github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2.1(0x2857180, 0xc4218fbeb0, 0xc4214782a0, 0xc4207ba600, 0xc420020748, 0xc67b3d)
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:287 +0x43
github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2()
/go/src/github.com/cockroachdb/cockroach/gossip/client.go:295 +0xb7
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4219f6fc0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17300 [select, 6 minutes]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421a1ea80, 0xc421a0efb0, 0x5, 0x5, 0x0, 0x0, 0x0)
/go/src/google.golang.org/grpc/transport/transport.go:141 +0x89c
google.golang.org/grpc/transport.(*Stream).Read(0xc4206ad2c0, 0xc421a0efb0, 0x5, 0x5, 0x61a0f7, 0xc420432070, 0xc420432054)
/go/src/google.golang.org/grpc/transport/transport.go:314 +0x79
io.ReadAtLeast(0x2841080, 0xc4206ad2c0, 0xc421a0efb0, 0x5, 0x5, 0x5, 0xc42119b7e8, 0x61a47a, 0xc420545380)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x2841080, 0xc4206ad2c0, 0xc421a0efb0, 0x5, 0x5, 0xc4ffffffff, 0x0, 0xc420432000)
/usr/local/go/src/io/io.go:325 +0x73
google.golang.org/grpc.(*parser).recvMsg(0xc421a0efa0, 0x7fffffff, 0x5e25a5, 0xc4218d9f18, 0x0, 0x0, 0x6, 0x0)
/go/src/google.golang.org/grpc/rpc_util.go:231 +0xa4
google.golang.org/grpc.recv(0xc421a0efa0, 0x284fac0, 0x354ff60, 0xc4206ad2c0, 0x0, 0x0, 0x1c03280, 0xc421a678c0, 0x7fffffff, 0x1, ...)
/go/src/google.golang.org/grpc/rpc_util.go:315 +0x5e
google.golang.org/grpc.(*serverStream).RecvMsg(0xc421a20f30, 0x1c03280, 0xc421a678c0, 0x0, 0x0)
/go/src/google.golang.org/grpc/stream.go:492 +0x154
github.com/cockroachdb/cockroach/gossip.(*gossipGossipServer).Recv(0xc420d30c80, 0xc4ffffffff, 0x0, 0x180002)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:230 +0x86
github.com/cockroachdb/cockroach/gossip.(Gossip_GossipServer).Recv-fm(0xc421478018, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x4b
github.com/cockroachdb/cockroach/gossip.(*server).gossipReceiver(0xc421478000, 0xc420026488, 0xc421a1eb40, 0xc42119bf40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:310 +0x604
github.com/cockroachdb/cockroach/gossip.(*server).Gossip.func3.1()
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:143 +0x116
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421a1eb80)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17430 [select, 6 minutes]:
google.golang.org/grpc/transport.(*http2Server).controller(0xc420cc5c20)
/go/src/google.golang.org/grpc/transport/http2_server.go:673 +0xa56
created by google.golang.org/grpc/transport.newHTTP2Server
/go/src/google.golang.org/grpc/transport/http2_server.go:139 +0xa91
goroutine 17406 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc420b0b320)
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x64
created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0xb61
goroutine 17587 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).maybeAddToPurgatory.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:524 +0xebd
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421670e80)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17435 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/gossip.(*server).start.func3()
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x94
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc421907820)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17874 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc42190d880, 0xecf57acee, 0xc41d73e246, 0x2b92920, 0xc4207dc200, 0xc421870b40, 0xc4218f3b00, 0xc4212673a0)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x299
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1.1(0xc4218f3b00, 0xc4210ac330)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:259 +0x124
github.com/cockroachdb/cockroach/storage.(*storeRangeSet).Visit(0xc4210ac840, 0xc421952100)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:243 +0x453
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:261 +0x3a5
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42038c3e0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17564 [select]:
github.com/cockroachdb/cockroach/gossip.(*server).Gossip(0xc42037c380, 0x2856880, 0xc421a23070, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/gossip/server.go:180 +0x73e
github.com/cockroachdb/cockroach/gossip._Gossip_Gossip_Handler(0x1be3a80, 0xc42037c380, 0x2852e00, 0xc420389050, 0x3a, 0x7f03334d9627)
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:211 +0xda
google.golang.org/grpc.(*Server).processStreamingRPC(0xc4213845a0, 0x2856e20, 0xc420cc4480, 0xc42116f950, 0xc4213eaf60, 0x28263a0, 0xc420b1aab0, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:689 +0xc7d
google.golang.org/grpc.(*Server).handleStream(0xc4213845a0, 0x2856e20, 0xc420cc4480, 0xc42116f950, 0xc420b1aab0)
/go/src/google.golang.org/grpc/server.go:773 +0x170e
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4218fb850, 0xc4213845a0, 0x2856e20, 0xc420cc4480, 0xc42116f950)
/go/src/google.golang.org/grpc/server.go:422 +0xb9
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/google.golang.org/grpc/server.go:423 +0xbb
goroutine 17246 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).Start.func3()
/go/src/github.com/cockroachdb/cockroach/storage/store.go:927 +0x19d
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a8c720)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17276 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/util/netutil.ListenAndServeGRPC.func1()
/go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:51 +0x86
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc4213b8060)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17255 [chan receive]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc4219b32c0)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17349 [select, 6 minutes]:
google.golang.org/grpc.NewClientStream.func2(0x2856dc0, 0xc421468200, 0xc4206ad590, 0xc421966c60)
/go/src/google.golang.org/grpc/stream.go:197 +0x54e
created by google.golang.org/grpc.NewClientStream
/go/src/google.golang.org/grpc/stream.go:217 +0x10a5
goroutine 17261 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc41b0, 0xc421a8cb40)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17894 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420af5080, 0xecf57acee, 0xc41e3184b2, 0x2b92920, 0xc4207dc200, 0xc421870cf0, 0xc421790000, 0xc4214a64a0)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x299
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1.1(0xc421790000, 0xc4210ad680)
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:259 +0x124
github.com/cockroachdb/cockroach/storage.(*storeRangeSet).Visit(0xc4210adb90, 0xc421952400)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:243 +0x453
github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/scanner.go:261 +0x3a5
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870cf0, 0xc42038cae0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17899 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870ea0, 0xc42038ce20)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17403 [select, 6 minutes]:
google.golang.org/grpc.NewClientStream.func2(0x2856dc0, 0xc421468200, 0xc4219ce960, 0xc420ca0fc0)
/go/src/google.golang.org/grpc/stream.go:197 +0x54e
created by google.golang.org/grpc.NewClientStream
/go/src/google.golang.org/grpc/stream.go:217 +0x10a5
goroutine 17478 [IO wait]:
net.runtime_pollWait(0x7f033355ae70, 0x72, 0x5ef973)
/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc420987250, 0x72, 0x5ec228, 0x1b63f20)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc420987250, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc4209871f0, 0xc4218ca000, 0x8000, 0x8000, 0x0, 0x2844c00, 0xc4200100c8)
/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc420026750, 0xc4218ca000, 0x8000, 0x8000, 0x0, 0x6, 0x2)
/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc420026750, 0xc4218ca000, 0x8000, 0x8000, 0x0, 0x7fd1, 0x1)
<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc4211a9920)
/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc4211a9920, 0xc420bf8db8, 0x9, 0x9, 0x10, 0x10, 0xc4207a7fa0)
/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x283d340, 0xc4211a9920, 0xc420bf8db8, 0x9, 0x9, 0x9, 0x61a47a, 0xc4212ca340, 0xc421f2c878)
/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x283d340, 0xc4211a9920, 0xc420bf8db8, 0x9, 0x9, 0x0, 0xc4207790e0, 0xc4214dca20)
/usr/local/go/src/io/io.go:325 +0x73
golang.org/x/net/http2.readFrameHeader(0xc420bf8db8, 0x9, 0x9, 0x283d340, 0xc4211a9920, 0x0, 0xc400000000, 0xc421f2c878, 0x28411c0)
/go/src/golang.org/x/net/http2/frame.go:237 +0xa3
golang.org/x/net/http2.(*Framer).ReadFrame(0xc420bf8d80, 0x0, 0x283d8c0, 0xc420010170, 0xc421f250e0)
/go/src/golang.org/x/net/http2/frame.go:464 +0x105
google.golang.org/grpc/transport.(*framer).readFrame(0xc420c1d620, 0xc421f2c960, 0xc421f2c960, 0xc4214dcce0, 0x1)
/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc421a21ef0, 0xc420c1d800)
/go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x30a
google.golang.org/grpc.(*Server).serveStreams(0xc421187040, 0x2856e20, 0xc421a21ef0)
/go/src/google.golang.org/grpc/server.go:424 +0x1d6
google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc421187040, 0x28573c0, 0xc420026750, 0x0, 0x0)
/go/src/google.golang.org/grpc/server.go:411 +0x4f1
google.golang.org/grpc.(*Server).handleRawConn(0xc421187040, 0x28573c0, 0xc420026750)
/go/src/google.golang.org/grpc/server.go:388 +0x5c3
created by google.golang.org/grpc.(*Server).Serve
/go/src/google.golang.org/grpc/server.go:357 +0x20a
goroutine 17882 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421140240)
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2657 +0x58
created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1
/go/src/github.com/cockroachdb/cockroach/storage/store.go:2660 +0xd8
goroutine 17378 [select]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1()
/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x416
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc42090f5b0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17224 [runnable]:
time.startTimer(0xc4217ef448)
/usr/local/go/src/runtime/time.go:68 +0x39
time.NewTimer(0x2540be400, 0xeb09bc)
/usr/local/go/src/time/sleep.go:84 +0x122
github.com/cockroachdb/cockroach/util/timeutil.(*Timer).Reset(0xc4218760b0, 0x2540be400)
/go/src/github.com/cockroachdb/cockroach/util/timeutil/timer.go:61 +0x147
github.com/cockroachdb/cockroach/kv.(*DistSender).sendToReplicas(0xc420cc42d0, 0x7f03335026f8, 0xc42125eea0, 0x2540be400, 0xb2d05e00, 0xc4218fa660, 0x1, 0xc4217f85e0, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:1018 +0x271
github.com/cockroachdb/cockroach/kv.(*DistSender).sendRPC(0xc420cc42d0, 0x7f03335026f8, 0xc42125eea0, 0x1, 0xc4217f85e0, 0x1, 0x1, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:379 +0x27c
github.com/cockroachdb/cockroach/kv.(*DistSender).RangeLookup(0xc420cc42d0, 0xc4201ebc0c, 0x2, 0x2, 0xc4217e97a0, 0xc421790001, 0xc4201ebc0c, 0x2, 0x2, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:265 +0x30f
github.com/cockroachdb/cockroach/storage_test.(*multiTestContext).RangeLookup(0xc420c4e340, 0xc4201ebc0c, 0x2, 0x2, 0xc4217e97a0, 0x1, 0x0, 0x1, 0xc4217e97a0, 0xc4217ef340, ...)
/go/src/github.com/cockroachdb/cockroach/storage/client_test.go:569 +0xf7
github.com/cockroachdb/cockroach/kv.(*rangeDescriptorCache).performRangeLookup(0xc4207dca00, 0x7f03335026f8, 0xc421793050, 0xc421179c28, 0x1, 0x8, 0x1, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/kv/range_cache.go:459 +0x4db
github.com/cockroachdb/cockroach/kv.(*rangeDescriptorCache).lookupRangeDescriptorInternal(0xc4207dca00, 0x7f03335026f8, 0xc421793050, 0xc421179c28, 0x1, 0x8, 0xc4217eed40, 0xc4217f0001, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/kv/range_cache.go:344 +0x678
github.com/cockroachdb/cockroach/kv.(*rangeDescriptorCache).LookupRangeDescriptor(0xc4207dca00, 0x7f03335026f8, 0xc421793050, 0xc421179c28, 0x1, 0x8, 0xc4217eed40, 0x1, 0xc4201ebc80, 0x1aef200, ...)
/go/src/github.com/cockroachdb/cockroach/kv/range_cache.go:282 +0xc0
github.com/cockroachdb/cockroach/kv.(*DistSender).getDescriptors(0xc420cc42d0, 0x7f03335026f8, 0xc421793050, 0xc421179c28, 0x1, 0x8, 0xc421179c28, 0x2, 0x8, 0xc4217eed40, ...)
/go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:444 +0x102
github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk(0xc420cc42d0, 0x7f03335026f8, 0xc421793050, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:689 +0x3ec
github.com/cockroachdb/cockroach/kv.(*DistSender).Send(0xc420cc42d0, 0x7f033353d240, 0xc420010580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:606 +0x1a1
github.com/cockroachdb/cockroach/internal/client.SendWrappedWith(0x283dc00, 0xc420cc42d0, 0x7f033353d240, 0xc420010580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/internal/client/sender.go:54 +0x191
github.com/cockroachdb/cockroach/internal/client.SendWrapped(0x283dc00, 0xc420cc42d0, 0x0, 0x0, 0x2857d60, 0xc42195ab80, 0x1, 0x8, 0x9)
/go/src/github.com/cockroachdb/cockroach/internal/client/sender.go:69 +0xc0
github.com/cockroachdb/cockroach/storage_test.TestSplitSnapshotRace_SplitWins(0xc420c10180)
/go/src/github.com/cockroachdb/cockroach/storage/client_split_test.go:1004 +0x368
testing.tRunner(0xc420c10180, 0x1f651b0)
/usr/local/go/src/testing/testing.go:610 +0xca
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:646 +0x530
goroutine 17520 [select, 6 minutes]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc4214b4780)
/go/src/google.golang.org/grpc/clientconn.go:720 +0x7d7
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
/go/src/google.golang.org/grpc/clientconn.go:467 +0x9c9
goroutine 17876 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42038c4e0)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17341 [select, 5 minutes]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc420409600)
/go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x8c9
created by google.golang.org/grpc/transport.newHTTP2Client
/go/src/google.golang.org/grpc/transport/http2_client.go:233 +0x14bd
goroutine 17226 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc4219c8420)
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x64
created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open
/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0xb61
goroutine 17879 [select, 6 minutes]:
github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1()
/go/src/github.com/cockroachdb/cockroach/storage/queue.go:398 +0x395
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc421870b40, 0xc42038c540)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
goroutine 17343 [select]:
github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc421aa8000, 0xc420ba7300, 0xc420ca4570, 0xf, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:282 +0x813
github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1()
/go/src/github.com/cockroachdb/cockroach/rpc/context.go:205 +0xb3
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420cc4090, 0xc420ea4360)
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74
FAIL github.com/cockroachdb/cockroach/storage 480.215s
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment