Skip to content

Instantly share code, notes, and snippets.

@benesch
Created August 31, 2017 18:12
Show Gist options
  • Select an option

  • Save benesch/a8b32409f9b76f7fe433215eb18021b8 to your computer and use it in GitHub Desktop.

Select an option

Save benesch/a8b32409f9b76f7fe433215eb18021b8 to your computer and use it in GitHub Desktop.
goroutine profile: total 336
32 @ 0x576e2a 0x576f0e 0x587e7b 0x5b9c49 0xfefe3c 0x102c10e 0xc953a7 0x5a44e1
# 0x587e7a sync.runtime_notifyListWait+0x10a /home/radu/go1.8/src/runtime/sema.go:297
# 0x5b9c48 sync.(*Cond).Wait+0x88 /home/radu/go1.8/src/sync/cond.go:57
# 0xfefe3b github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker+0xab /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:198
# 0x102c10d github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2+0x3d /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:167
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
32 @ 0x576e2a 0x586234 0x584e9c 0x143e014 0xc953a7 0x5a44e1
# 0x143e013 github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1+0x183 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
27 @ 0x576e2a 0x576f0e 0x587e7b 0x5b9c49 0x123aa03 0x5a44e1
# 0x587e7a sync.runtime_notifyListWait+0x10a /home/radu/go1.8/src/runtime/sema.go:297
# 0x5b9c48 sync.(*Cond).Wait+0x88 /home/radu/go1.8/src/sync/cond.go:57
# 0x123aa02 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*routerBase).start.func1+0x352 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/routers.go:250
24 @ 0x576e2a 0x586234 0x584e9c 0x9feaa9 0x5a44e1
# 0x9feaa8 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func3+0x3c8 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:255
21 @ 0x576e2a 0x586234 0x584e9c 0x1205c1a 0x1206333 0x5a44e1
# 0x1205c19 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*outbox).mainLoop+0x689 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/outbox.go:222
# 0x1206332 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*outbox).run+0x42 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/outbox.go:355
21 @ 0x576e2a 0x586234 0x584e9c 0x9d9f15 0x9dae25 0x5bb739 0x5bb8a8 0x9dad78 0x9ecf15 0x9edbfd 0x9f9622 0x11c62e2 0x123a465 0xc954e6 0x5a44e1
# 0x9d9f14 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read+0x494 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:126
# 0x9dae24 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read+0x54 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:391
# 0x5bb738 io.ReadAtLeast+0xa8 /home/radu/go1.8/src/io/io.go:307
# 0x5bb8a7 io.ReadFull+0x57 /home/radu/go1.8/src/io/io.go:325
# 0x9dad77 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read+0xc7 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:375
# 0x9ecf14 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg+0x64 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:265
# 0x9edbfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv+0x4c /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:361
# 0x9f9621 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg+0xf1 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:398
# 0x11c62e1 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*distSQLFlowStreamClient).Recv+0x61 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/api.pb.go:289
# 0x123a464 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*outbox).listenForDrainSignalFromConsumer.func1+0x94 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/outbox.go:326
# 0xc954e5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1+0xe5 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:269
15 @ 0x576e2a 0x586234 0x584e9c 0x9d9f15 0x9dae25 0x5bb739 0x5bb8a8 0x9dad78 0x9ecf15 0x9edbfd 0x9faf0a 0x11c6ac2 0x11fcfc1 0x11fc72b 0x122e124 0x122e3d5 0x11c69a7 0x9f4d6c 0x9f674f 0x9fd359 0x5a44e1
# 0x9d9f14 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read+0x494 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:126
# 0x9dae24 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read+0x54 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:391
# 0x5bb738 io.ReadAtLeast+0xa8 /home/radu/go1.8/src/io/io.go:307
# 0x5bb8a7 io.ReadFull+0x57 /home/radu/go1.8/src/io/io.go:325
# 0x9dad77 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read+0xc7 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:375
# 0x9ecf14 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg+0x64 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:265
# 0x9edbfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv+0x4c /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:361
# 0x9faf09 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg+0x119 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:648
# 0x11c6ac1 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*distSQLFlowStreamServer).Recv+0x61 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/api.pb.go:386
# 0x11fcfc0 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.processInboundStreamHelper+0x670 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/inbound.go:81
# 0x11fc72a github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.ProcessInboundStream+0x9a /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/inbound.go:39
# 0x122e123 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*ServerImpl).flowStreamInt+0x433 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/server.go:423
# 0x122e3d4 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*ServerImpl).FlowStream+0xa4 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/server.go:429
# 0x11c69a6 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun._DistSQL_FlowStream_Handler+0xb6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/api.pb.go:367
# 0x9f4d6b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC+0xa4b /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:891
# 0x9f674e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream+0x153e /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:985
# 0x9fd358 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1+0xa8 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:551
12 @ 0x576e2a 0x576f0e 0x54e36d 0x54e07d 0x11cfec8 0x11d0434 0x11fcc17 0x11fc72b 0x122e124 0x122e3d5 0x11c69a7 0x9f4d6c 0x9f674f 0x9fd359 0x5a44e1
# 0x11cfec7 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*RowChannel).Push+0x1d7 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/base.go:331
# 0x11d0433 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*MultiplexedRowChannel).Push+0x73 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/base.go:411
# 0x11fcc16 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.processInboundStreamHelper+0x2c6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/inbound.go:114
# 0x11fc72a github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.ProcessInboundStream+0x9a /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/inbound.go:39
# 0x122e123 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*ServerImpl).flowStreamInt+0x433 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/server.go:423
# 0x122e3d4 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*ServerImpl).FlowStream+0xa4 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/server.go:429
# 0x11c69a6 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun._DistSQL_FlowStream_Handler+0xb6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/api.pb.go:367
# 0x9f4d6b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC+0xa4b /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:891
# 0x9f674e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream+0x153e /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:985
# 0x9fd358 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1+0xa8 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:551
11 @ 0x576e2a 0x571f47 0x571589 0x6fc5c8 0x6fc634 0x6fdd87 0x711210 0x87abe0 0x650147 0x6510cb 0x6512b7 0x7fffaf 0x7ffdaf 0x874bd5 0x87bfb3 0x8803ba 0x5a44e1
# 0x571588 net.runtime_pollWait+0x58 /home/radu/go1.8/src/runtime/netpoll.go:164
# 0x6fc5c7 net.(*pollDesc).wait+0x37 /home/radu/go1.8/src/net/fd_poll_runtime.go:75
# 0x6fc633 net.(*pollDesc).waitRead+0x33 /home/radu/go1.8/src/net/fd_poll_runtime.go:80
# 0x6fdd86 net.(*netFD).Read+0x1b6 /home/radu/go1.8/src/net/fd_unix.go:250
# 0x71120f net.(*conn).Read+0x6f /home/radu/go1.8/src/net/net.go:181
# 0x87abdf net/http.(*connReader).Read+0x13f /home/radu/go1.8/src/net/http/server.go:754
# 0x650146 bufio.(*Reader).fill+0x116 /home/radu/go1.8/src/bufio/bufio.go:97
# 0x6510ca bufio.(*Reader).ReadSlice+0xba /home/radu/go1.8/src/bufio/bufio.go:338
# 0x6512b6 bufio.(*Reader).ReadLine+0x36 /home/radu/go1.8/src/bufio/bufio.go:367
# 0x7fffae net/textproto.(*Reader).readLineSlice+0x5e /home/radu/go1.8/src/net/textproto/reader.go:55
# 0x7ffdae net/textproto.(*Reader).ReadLine+0x2e /home/radu/go1.8/src/net/textproto/reader.go:36
# 0x874bd4 net/http.readRequest+0xa4 /home/radu/go1.8/src/net/http/request.go:918
# 0x87bfb2 net/http.(*conn).readRequest+0x212 /home/radu/go1.8/src/net/http/server.go:934
# 0x8803b9 net/http.(*conn).serve+0x499 /home/radu/go1.8/src/net/http/server.go:1763
8 @ 0x576e2a 0x586234 0x584e9c 0x101ed6f 0xc953a7 0x5a44e1
# 0x101ed6e github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1+0x42e /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:504
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
7 @ 0x576e2a 0x576f0e 0x54e36d 0x54e07d 0x14a69e9 0x75fbe7 0x5a44e1
# 0x14a69e8 github.com/cockroachdb/cockroach/pkg/ccl/sqlccl.splitAndScatter.func2+0x688 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/ccl/sqlccl/restore.go:643
# 0x75fbe6 github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup.(*Group).Go.func1+0x56 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:58
5 @ 0x576e2a 0x586234 0x584e9c 0x9da7a7 0x9e298a 0x9e44ad 0xd4f0a3 0x9e33fd 0xb55102 0x11a4a8f 0x11a525d 0x5a44e1
# 0x9da7a6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header+0x2f6 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:290
# 0x9e2989 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse+0xa9 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:49
# 0x9e44ac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke+0xfbc /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:282
# 0xd4f0a2 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1+0xf82 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44
# 0x9e33fc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke+0xdc /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:135
# 0xb55101 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch+0xd1 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3487
# 0x11a4a8e github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1+0x17e /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:228
# 0x11a525c github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1+0xac /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:237
4 @ 0x576e2a 0x571f47 0x571589 0x6fc5c8 0x6fc634 0x6fdd87 0x711210 0x650962 0x5bb739 0x5bb8a8 0x997a5b 0x9983b4 0x9d943f 0x9cbb0d 0x5a44e1
# 0x571588 net.runtime_pollWait+0x58 /home/radu/go1.8/src/runtime/netpoll.go:164
# 0x6fc5c7 net.(*pollDesc).wait+0x37 /home/radu/go1.8/src/net/fd_poll_runtime.go:75
# 0x6fc633 net.(*pollDesc).waitRead+0x33 /home/radu/go1.8/src/net/fd_poll_runtime.go:80
# 0x6fdd86 net.(*netFD).Read+0x1b6 /home/radu/go1.8/src/net/fd_unix.go:250
# 0x71120f net.(*conn).Read+0x6f /home/radu/go1.8/src/net/net.go:181
# 0x650961 bufio.(*Reader).Read+0x311 /home/radu/go1.8/src/bufio/bufio.go:213
# 0x5bb738 io.ReadAtLeast+0xa8 /home/radu/go1.8/src/io/io.go:307
# 0x5bb8a7 io.ReadFull+0x57 /home/radu/go1.8/src/io/io.go:325
# 0x997a5a github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader+0x7a /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237
# 0x9983b3 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame+0xa3 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492
# 0x9d943e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame+0x2e /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:592
# 0x9cbb0c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader+0xcc /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1154
4 @ 0x576e2a 0x586234 0x584e9c 0x9cca87 0x5a44e1
# 0x9cca86 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller+0x6c6 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1227
4 @ 0x576e2a 0x586234 0x584e9c 0x9ccf8d 0x5a44e1
# 0x9ccf8c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive+0x4ec /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1276
3 @ 0x576e2a 0x571f47 0x571589 0x6fc5c8 0x6fc634 0x6fdd87 0x711210 0xd53063 0x650962 0x5bb739 0x5bb8a8 0x997a5b 0x9983b4 0x9d943f 0x9cfcdc 0x9f1302 0x9f1157 0x9f0c7d 0x5a44e1
# 0x571588 net.runtime_pollWait+0x58 /home/radu/go1.8/src/runtime/netpoll.go:164
# 0x6fc5c7 net.(*pollDesc).wait+0x37 /home/radu/go1.8/src/net/fd_poll_runtime.go:75
# 0x6fc633 net.(*pollDesc).waitRead+0x33 /home/radu/go1.8/src/net/fd_poll_runtime.go:80
# 0x6fdd86 net.(*netFD).Read+0x1b6 /home/radu/go1.8/src/net/fd_unix.go:250
# 0x71120f net.(*conn).Read+0x6f /home/radu/go1.8/src/net/net.go:181
# 0xd53062 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read+0x112 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218
# 0x650961 bufio.(*Reader).Read+0x311 /home/radu/go1.8/src/bufio/bufio.go:213
# 0x5bb738 io.ReadAtLeast+0xa8 /home/radu/go1.8/src/io/io.go:307
# 0x5bb8a7 io.ReadFull+0x57 /home/radu/go1.8/src/io/io.go:325
# 0x997a5a github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader+0x7a /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237
# 0x9983b3 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame+0xa3 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492
# 0x9d943e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame+0x2e /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:592
# 0x9cfcdb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams+0x32b /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:377
# 0x9f1301 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams+0x161 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:559
# 0x9f1156 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport+0x496 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:540
# 0x9f0c7c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn+0x4ac /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:507
3 @ 0x576e2a 0x576f0e 0x54e36d 0x54e07d 0x11cfec8 0x1207c1a 0x1234fd2 0x5a44e1
# 0x11cfec7 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*RowChannel).Push+0x1d7 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/base.go:331
# 0x1207c19 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*ProcOutputHelper).EmitRow+0x269 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/processors.go:302
# 0x1234fd1 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*tableReader).Run+0x411 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/tablereader.go:214
3 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f045 0x11cffe0 0x11c2449 0x11c1c31 0x5a44e1
# 0x11cffdf github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*RowChannel).Next+0x9f /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/base.go:355
# 0x11c2448 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*aggregator).accumulateRows+0x118 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/aggregator.go:257
# 0x11c1c30 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*aggregator).Run+0x1e0 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/aggregator.go:195
3 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f045 0x11cffe0 0x11cf942 0x122faa7 0x122f3d2 0x122eec9 0x5a44e1
# 0x11cffdf github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*RowChannel).Next+0x9f /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/base.go:355
# 0x11cf941 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*NoMetadataRowSource).NextRow+0x51 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/base.go:254
# 0x122faa6 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*sortAllStrategy).executeImpl+0x76 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/sorterstrategy.go:121
# 0x122f3d1 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*sortAllStrategy).Execute+0xe1 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/sorterstrategy.go:62
# 0x122eec8 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*sorter).Run+0x618 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/sorter.go:148
3 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f045 0x11d05f3 0x11c2449 0x11c1c31 0x5a44e1
# 0x11d05f2 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*MultiplexedRowChannel).Next+0xc2 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/base.go:432
# 0x11c2448 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*aggregator).accumulateRows+0x118 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/aggregator.go:257
# 0x11c1c30 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*aggregator).Run+0x1e0 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/aggregator.go:195
3 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f045 0x11d05f3 0x11f9f59 0x11faef6 0x11fa429 0x11f9399 0x5a44e1
# 0x11d05f2 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*MultiplexedRowChannel).Next+0xc2 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/base.go:432
# 0x11f9f58 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*hashJoiner).receiveRow+0x58 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/hashjoiner.go:269
# 0x11faef5 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*hashJoiner).bufferPhaseImpl+0x185 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/hashjoiner.go:417
# 0x11fa428 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*hashJoiner).bufferPhase+0x88 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/hashjoiner.go:313
# 0x11f9398 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*hashJoiner).Run+0x7c8 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/hashjoiner.go:205
3 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f045 0x11d05f3 0x11f9f59 0x11fb426 0x11fa429 0x11f9399 0x5a44e1
# 0x11d05f2 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*MultiplexedRowChannel).Next+0xc2 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/base.go:432
# 0x11f9f58 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*hashJoiner).receiveRow+0x58 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/hashjoiner.go:269
# 0x11fb425 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*hashJoiner).bufferPhaseImpl+0x6b5 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/hashjoiner.go:462
# 0x11fa428 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*hashJoiner).bufferPhase+0x88 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/hashjoiner.go:313
# 0x11f9398 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*hashJoiner).Run+0x7c8 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/hashjoiner.go:205
3 @ 0x576e2a 0x576f0e 0x587721 0x587324 0x5bb2ba 0x1238333 0x5a44e1
# 0x587323 sync.runtime_Semacquire+0x33 /home/radu/go1.8/src/runtime/sema.go:47
# 0x5bb2b9 sync.(*WaitGroup).Wait+0x79 /home/radu/go1.8/src/sync/waitgroup.go:131
# 0x1238332 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*Flow).Wait.func1+0x32 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow.go:405
3 @ 0x576e2a 0x586234 0x584e9c 0x102df4e 0xc953a7 0x5a44e1
# 0x102df4d github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4+0x4ed /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1292
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
3 @ 0x576e2a 0x586234 0x584e9c 0x11940a8 0x118d49b 0x118d97b 0x1190de7 0x118fc16 0x118e7f4 0xd119ef 0xd190b3 0x1107be7 0x110883e 0x111012e 0x1110072 0x110fee7 0x1202f69 0x12034f6 0x5a44e1
# 0x11940a7 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas+0x15e7 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1144
# 0x118d49a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC+0x2da /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:382
# 0x118d97a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange+0x17a /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:446
# 0x1190de6 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch+0x446 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:930
# 0x118fc15 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges+0xb45 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:795
# 0x118e7f3 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send+0x343 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:597
# 0xd119ee github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send+0x1fe /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:545
# 0xd190b2 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send+0x342 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:908
# 0x1107be6 github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*txnKVFetcher).fetch+0x286 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/kvfetcher.go:229
# 0x110883d github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*txnKVFetcher).nextKV+0x2cd /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/kvfetcher.go:303
# 0x111012d github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowFetcher).NextKey+0x5d /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/rowfetcher.go:227
# 0x1110071 github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowFetcher).StartScanFrom+0xa1 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/rowfetcher.go:216
# 0x110fee6 github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowFetcher).StartScan+0x1e6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/rowfetcher.go:207
# 0x1202f68 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*joinReader).mainLoop+0x698 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/joinreader.go:160
# 0x12034f5 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*joinReader).Run+0x105 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/joinreader.go:205
3 @ 0x576e2a 0x586234 0x584e9c 0x11eebd8 0x12389cb 0x5a44e1
# 0x11eebd7 github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*Flow).Wait+0x187 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow.go:409
# 0x12389ca github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).runFlowNow.func2+0x2a /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:83
3 @ 0x576e2a 0x586234 0x584e9c 0x9d4d58 0x5a44e1
# 0x9d4d57 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive+0x487 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:949
3 @ 0x576e2a 0x586234 0x584e9c 0x9d56d6 0x5a44e1
# 0x9d56d5 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).controller+0x7d5 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:1011
3 @ 0x576e2a 0x586234 0x584e9c 0x9e9d54 0x9fcad9 0x5a44e1
# 0x9e9d53 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor+0x693 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:927
# 0x9fcad8 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1+0x1d8 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:642
3 @ 0x576e2a 0x586234 0x584e9c 0xd5acef 0xd5fb0b 0xc953a7 0x5a44e1
# 0xd5acee github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat+0x7ee /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:419
# 0xd5fb0a github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.2.1+0x6a /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:353
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
2 @ 0x576e2a 0x576f0e 0x587721 0x587324 0x5bb2ba 0x75fa51 0x1495692 0x14a7583 0x75fbe7 0x5a44e1
# 0x587323 sync.runtime_Semacquire+0x33 /home/radu/go1.8/src/runtime/sema.go:47
# 0x5bb2b9 sync.(*WaitGroup).Wait+0x79 /home/radu/go1.8/src/sync/waitgroup.go:131
# 0x75fa50 github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup.(*Group).Wait+0x30 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:41
# 0x1495691 github.com/cockroachdb/cockroach/pkg/ccl/sqlccl.splitAndScatter+0x651 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/ccl/sqlccl/restore.go:650
# 0x14a7582 github.com/cockroachdb/cockroach/pkg/ccl/sqlccl.restore.func2+0x112 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/ccl/sqlccl/restore.go:845
# 0x75fbe6 github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup.(*Group).Go.func1+0x56 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:58
2 @ 0x576e2a 0x576f0e 0x587e7b 0x5b9c49 0xce6257 0x5a44e1
# 0x587e7a sync.runtime_notifyListWait+0x10a /home/radu/go1.8/src/runtime/sema.go:297
# 0x5b9c48 sync.(*Cond).Wait+0x88 /home/radu/go1.8/src/sync/cond.go:57
# 0xce6256 github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop+0xd6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:482
2 @ 0x576e2a 0x586234 0x584e9c 0x11940a8 0x118d49b 0x118d97b 0x1190de7 0x11a32eb 0xc956a0 0x5a44e1
# 0x11940a7 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas+0x15e7 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1144
# 0x118d49a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC+0x2da /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:382
# 0x118d97a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange+0x17a /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:446
# 0x1190de6 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch+0x446 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:930
# 0x11a32ea github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatchAsync.func1+0x11a /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:862
# 0xc9569f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1+0x10f /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330
2 @ 0x576e2a 0x586234 0x584e9c 0x9d9f15 0x9dae25 0x5bb739 0x5bb8a8 0x9dad78 0x9ecf15 0x9edbfd 0x9f9622 0xf7c122 0x1021425 0x1021636 0xc953a7 0x5a44e1
# 0x9d9f14 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read+0x494 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:126
# 0x9dae24 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read+0x54 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:391
# 0x5bb738 io.ReadAtLeast+0xa8 /home/radu/go1.8/src/io/io.go:307
# 0x5bb8a7 io.ReadFull+0x57 /home/radu/go1.8/src/io/io.go:325
# 0x9dad77 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read+0xc7 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:375
# 0x9ecf14 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg+0x64 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:265
# 0x9edbfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv+0x4c /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:361
# 0x9f9621 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg+0xf1 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:398
# 0xf7c121 github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv+0x61 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:329
# 0x1021424 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1+0x34 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:487
# 0x1021635 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1+0x65 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:502
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
2 @ 0x576e2a 0x586234 0x584e9c 0x9d9f15 0x9dae25 0x5bb739 0x5bb8a8 0x9dad78 0x9ecf15 0x9edbfd 0x9faf0a 0xf7c6b2 0x10207b7 0x10209fd 0xc953a7 0x5a44e1
# 0x9d9f14 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read+0x494 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:126
# 0x9dae24 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read+0x54 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:391
# 0x5bb738 io.ReadAtLeast+0xa8 /home/radu/go1.8/src/io/io.go:307
# 0x5bb8a7 io.ReadFull+0x57 /home/radu/go1.8/src/io/io.go:325
# 0x9dad77 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read+0xc7 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:375
# 0x9ecf14 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg+0x64 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:265
# 0x9edbfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv+0x4c /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:361
# 0x9faf09 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg+0x119 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:648
# 0xf7c6b1 github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv+0x61 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:397
# 0x10207b6 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1+0x96 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:326
# 0x10209fc github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1+0x5c /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:349
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
2 @ 0x576e2a 0x586234 0x584e9c 0xf8a19d 0xf7c597 0x9f4d6c 0x9f674f 0x9fd359 0x5a44e1
# 0xf8a19c github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch+0x23c /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:355
# 0xf7c596 github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler+0xb6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:378
# 0x9f4d6b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC+0xa4b /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:891
# 0x9f674e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream+0x153e /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:985
# 0x9fd358 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1+0xa8 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:551
2 @ 0x576e2a 0x586234 0x584e9c 0xf8afb6 0x10212ee 0xf8a7c9 0x1021890 0xc953a7 0x5a44e1
# 0xf8afb5 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue+0x6a5 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:513
# 0x10212ed github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1+0x32d /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:458
# 0xf8a7c8 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess+0xe8 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:459
# 0x102188f github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func1.1+0x5f /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:587
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x558a62 0x58b024 0xca05a2 0x5a44e1
# 0x58b023 os/signal.signal_recv+0x103 /home/radu/go1.8/src/runtime/sigqueue.go:116
# 0xca05a1 os/signal.loop+0x21 /home/radu/go1.8/src/os/signal/signal_unix.go:22
1 @ 0x576e2a 0x571f47 0x571589 0x6fc5c8 0x6fc634 0x6fdd87 0x711210 0x87a6e8 0x5a44e1
# 0x571588 net.runtime_pollWait+0x58 /home/radu/go1.8/src/runtime/netpoll.go:164
# 0x6fc5c7 net.(*pollDesc).wait+0x37 /home/radu/go1.8/src/net/fd_poll_runtime.go:75
# 0x6fc633 net.(*pollDesc).waitRead+0x33 /home/radu/go1.8/src/net/fd_poll_runtime.go:80
# 0x6fdd86 net.(*netFD).Read+0x1b6 /home/radu/go1.8/src/net/fd_unix.go:250
# 0x71120f net.(*conn).Read+0x6f /home/radu/go1.8/src/net/net.go:181
# 0x87a6e7 net/http.(*connReader).backgroundRead+0x57 /home/radu/go1.8/src/net/http/server.go:656
1 @ 0x576e2a 0x571f47 0x571589 0x6fc5c8 0x6fc634 0x6ff5a5 0x71be5e 0x71a329 0x884628 0x1557d32 0xc953a7 0x5a44e1
# 0x571588 net.runtime_pollWait+0x58 /home/radu/go1.8/src/runtime/netpoll.go:164
# 0x6fc5c7 net.(*pollDesc).wait+0x37 /home/radu/go1.8/src/net/fd_poll_runtime.go:75
# 0x6fc633 net.(*pollDesc).waitRead+0x33 /home/radu/go1.8/src/net/fd_poll_runtime.go:80
# 0x6ff5a4 net.(*netFD).accept+0x1e4 /home/radu/go1.8/src/net/fd_unix.go:430
# 0x71be5d net.(*TCPListener).accept+0x2d /home/radu/go1.8/src/net/tcpsock_posix.go:136
# 0x71a328 net.(*TCPListener).Accept+0x48 /home/radu/go1.8/src/net/tcpsock.go:228
# 0x884627 net/http.(*Server).Serve+0x227 /home/radu/go1.8/src/net/http/server.go:2643
# 0x1557d31 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5+0x41 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:663
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x571f47 0x571589 0x6fc5c8 0x6fc634 0x6ff5a5 0x71be5e 0x71a329 0xd526f5 0x15587ff 0x5ba10e 0x1558894 0xc953a7 0x5a44e1
# 0x571588 net.runtime_pollWait+0x58 /home/radu/go1.8/src/runtime/netpoll.go:164
# 0x6fc5c7 net.(*pollDesc).wait+0x37 /home/radu/go1.8/src/net/fd_poll_runtime.go:75
# 0x6fc633 net.(*pollDesc).waitRead+0x33 /home/radu/go1.8/src/net/fd_poll_runtime.go:80
# 0x6ff5a4 net.(*netFD).accept+0x1e4 /home/radu/go1.8/src/net/fd_unix.go:430
# 0x71be5d net.(*TCPListener).accept+0x2d /home/radu/go1.8/src/net/tcpsock_posix.go:136
# 0x71a328 net.(*TCPListener).Accept+0x48 /home/radu/go1.8/src/net/tcpsock.go:228
# 0xd526f4 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve+0x94 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124
# 0x15587fe github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10.1+0x2e /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:834
# 0x5ba10d sync.(*Once).Do+0xbd /home/radu/go1.8/src/sync/once.go:44
# 0x1558893 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10+0x63 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:835
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f005 0x102c065 0xc953a7 0x5a44e1
# 0x102c064 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1+0x54 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:157
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f005 0x15579cf 0xc953a7 0x5a44e1
# 0x15579ce github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func2+0x5e /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:632
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f005 0x1557e48 0xc953a7 0x5a44e1
# 0x1557e47 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6+0x87 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:667
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f005 0xd55360 0xc953a7 0x5a44e1
# 0xd5535f github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2+0x6f /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f005 0xd5f92d 0xc953a7 0x5a44e1
# 0xd5f92c github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1+0x5c /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:216
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f005 0xd83c26 0xc953a7 0x5a44e1
# 0xd83c25 github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3+0x65 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:384
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f045 0xab44b7 0x5a44e1
# 0xab44b6 github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).flushDaemon+0x76 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:990
1 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f045 0xab46ef 0x5a44e1
# 0xab46ee github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon+0x5e /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1024
1 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f045 0xd53f6a 0x9f0326 0x1557f73 0xc953a7 0x5a44e1
# 0xd53f69 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept+0x79 <autogenerated>:9
# 0x9f0325 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve+0x195 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:444
# 0x1557f72 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func7+0x42 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:679
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x576f0e 0x54f3d1 0x54f045 0xd53f6a 0xd54d20 0x15586b4 0xc953a7 0x5a44e1
# 0xd53f69 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept+0x79 <autogenerated>:9
# 0xd54d1f github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith+0x4f /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:119
# 0x15586b3 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func9+0x1b3 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:743
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x576f0e 0x587721 0x587324 0x5bb2ba 0x75fa51 0x14976c3 0x1498f06 0x14a7c97 0x1445bea 0x5a44e1
# 0x587323 sync.runtime_Semacquire+0x33 /home/radu/go1.8/src/runtime/sema.go:47
# 0x5bb2b9 sync.(*WaitGroup).Wait+0x79 /home/radu/go1.8/src/sync/waitgroup.go:131
# 0x75fa50 github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup.(*Group).Wait+0x30 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:41
# 0x14976c2 github.com/cockroachdb/cockroach/pkg/ccl/sqlccl.restore+0x1652 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/ccl/sqlccl/restore.go:882
# 0x1498f05 github.com/cockroachdb/cockroach/pkg/ccl/sqlccl.doRestorePlan+0x565 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/ccl/sqlccl/restore.go:1038
# 0x14a7c96 github.com/cockroachdb/cockroach/pkg/ccl/sqlccl.restorePlanHook.func1+0x206 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/ccl/sqlccl/restore.go:987
# 0x1445be9 github.com/cockroachdb/cockroach/pkg/sql.(*hookFnNode).Start.func1+0x49 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/planhook.go:84
1 @ 0x576e2a 0x576f0e 0x587721 0x587324 0x5bb2ba 0x75fa51 0x14976c3 0x14a827e 0x11bf039 0x5a44e1
# 0x587323 sync.runtime_Semacquire+0x33 /home/radu/go1.8/src/runtime/sema.go:47
# 0x5bb2b9 sync.(*WaitGroup).Wait+0x79 /home/radu/go1.8/src/sync/waitgroup.go:131
# 0x75fa50 github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup.(*Group).Wait+0x30 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/errgroup/errgroup.go:41
# 0x14976c2 github.com/cockroachdb/cockroach/pkg/ccl/sqlccl.restore+0x1652 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/ccl/sqlccl/restore.go:882
# 0x14a827d github.com/cockroachdb/cockroach/pkg/ccl/sqlccl.restoreResumeHook.func1+0x40d /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/ccl/sqlccl/restore.go:1087
# 0x11bf038 github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).maybeAdoptJob.func2+0xf8 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:294
1 @ 0x576e2a 0x576f0e 0x587721 0x587324 0x5bb2ba 0xfefd71 0x10377ae 0xc953a7 0x5a44e1
# 0x587323 sync.runtime_Semacquire+0x33 /home/radu/go1.8/src/runtime/sema.go:47
# 0x5bb2b9 sync.(*WaitGroup).Wait+0x79 /home/radu/go1.8/src/sync/waitgroup.go:131
# 0xfefd70 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait+0x30 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:173
# 0x10377ad github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm+0x3d /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3595
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x576f0e 0x587721 0x587404 0x5b9e9d 0x5ba094 0x1558974 0xc953a7 0x5a44e1
# 0x587403 sync.runtime_SemacquireMutex+0x33 /home/radu/go1.8/src/runtime/sema.go:62
# 0x5b9e9c sync.(*Mutex).Lock+0x9c /home/radu/go1.8/src/sync/mutex.go:87
# 0x5ba093 sync.(*Once).Do+0x43 /home/radu/go1.8/src/sync/once.go:40
# 0x1558973 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func11+0x63 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:922
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x1007a48 0x10377fe 0xc953a7 0x5a44e1
# 0x1007a47 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop+0x227 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3611
# 0x10377fd github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.raftTickLoop)-fm+0x3d /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3597
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x1007c06 0x103784e 0xc953a7 0x5a44e1
# 0x1007c05 github.com/cockroachdb/cockroach/pkg/storage.(*Store).coalescedHeartbeatsLoop+0x185 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3652
# 0x103784d github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.coalescedHeartbeatsLoop)-fm+0x3d /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3598
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x101c22e 0xc953a7 0x5a44e1
# 0x101c22d github.com/cockroachdb/cockroach/pkg/storage.(*idAllocator).start.func1+0x73d /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/id_alloc.go:134
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x101d026 0xc953a7 0x5a44e1
# 0x101d025 github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1+0x285 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:351
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x102d7c5 0xc953a7 0x5a44e1
# 0x102d7c4 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func2+0x164 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1163
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x119d47d 0x11a53ce 0xc953a7 0x5a44e1
# 0x119d47c github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).printStatsLoop+0xb4c /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:253
# 0x11a53cd github.com/cockroachdb/cockroach/pkg/kv.NewTxnCoordSender.func1+0x3d /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:237
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x11bea6c 0xc953a7 0x5a44e1
# 0x11bea6b github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func1+0x18b /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:129
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x11beca7 0xc953a7 0x5a44e1
# 0x11beca6 github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func2+0x226 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:140
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x123906f 0xc953a7 0x5a44e1
# 0x123906e github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1+0x48e /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:126
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x13c6d1b 0x1370352 0x13712fb 0x136e500 0x136c810 0x136be29 0x143e884 0xd18378 0x136a7bf 0x136a1b9 0x1369679 0x1501738 0x1500bdb 0x14f73cc 0x1558439 0xd554c9 0x5a44e1
# 0x13c6d1a github.com/cockroachdb/cockroach/pkg/sql.(*hookFnNode).Next+0x1ea /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/planhook.go:92
# 0x1370351 github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execClassic+0x271 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:1739
# 0x13712fa github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execStmt+0x62a /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:1869
# 0x136e4ff github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execStmtInOpenTxn+0x6ef /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:1582
# 0x136c80f github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execStmtsInCurrentTxn+0x8df /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:1153
# 0x136be28 github.com/cockroachdb/cockroach/pkg/sql.runTxnAttempt+0xc8 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:1033
# 0x143e883 github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execParsed.func1+0x293 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:841
# 0xd18377 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Exec+0xa7 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:722
# 0x136a7be github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execParsed+0x47e /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:862
# 0x136a1b8 github.com/cockroachdb/cockroach/pkg/sql.(*Executor).execRequest+0x488 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:747
# 0x1369678 github.com/cockroachdb/cockroach/pkg/sql.(*Executor).ExecuteStatements+0x128 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:629
# 0x1501737 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*v3Conn).handleSimpleQuery+0x147 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:579
# 0x1500bda github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*v3Conn).serve+0x8ba /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:497
# 0x14f73cb github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn+0x96b /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:421
# 0x1558438 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func9.1+0x148 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:737
# 0xd554c8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1+0xa8 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140
1 @ 0x576e2a 0x586234 0x584e9c 0x143e5c5 0xc953a7 0x5a44e1
# 0x143e5c4 github.com/cockroachdb/cockroach/pkg/sql.(*Executor).Start.func1+0x164 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:421
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x1445415 0xc953a7 0x5a44e1
# 0x1445414 github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1+0xaf4 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1362
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x144a95a 0xc953a7 0x5a44e1
# 0x144a959 github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1+0x1639 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:974
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x14c947f 0xc953a7 0x5a44e1
# 0x14c947e github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1+0x16e /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:88
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x1555fe4 0xc953a7 0x5a44e1
# 0x1555fe3 github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1+0x463 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:659
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x1556403 0xc953a7 0x5a44e1
# 0x1556402 github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1+0x222 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:700
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x1556822 0xc953a7 0x5a44e1
# 0x1556821 github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteSummaries.func1+0x2c1 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:737
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x1559785 0xc953a7 0x5a44e1
# 0x1559784 github.com/cockroachdb/cockroach/pkg/server.(*Server).startSampleEnvironment.func1+0x184 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1167
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x155a59e 0xc953a7 0x5a44e1
# 0x155a59d github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2+0x38d /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:112
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x155cdb6 0xc953a7 0x5a44e1
# 0x155cdb5 github.com/cockroachdb/cockroach/pkg/server.(*Server).PeriodicallyCheckForUpdates.func1+0x465 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/updates.go:121
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x15c2c2c 0x15ce2da 0x15ce4b1 0x14b8b28 0x14b9339 0x14b8fdb 0x1599d7d 0x159998d 0x15d75a0 0x5769da 0x5a44e1
# 0x15c2c2b github.com/cockroachdb/cockroach/pkg/cli.runStart+0xedb /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/cli/start.go:440
# 0x15ce2d9 github.com/cockroachdb/cockroach/pkg/cli.MaybeDecorateGRPCError.func1+0x59 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/cli/error.go:35
# 0x15ce4b0 github.com/cockroachdb/cockroach/pkg/cli.MaybeShoutError.func1+0x50 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/cli/error.go:66
# 0x14b8b27 github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra.(*Command).execute+0x3f7 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:644
# 0x14b9338 github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra.(*Command).ExecuteC+0x338 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:734
# 0x14b8fda github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra.(*Command).Execute+0x2a /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:693
# 0x1599d7c github.com/cockroachdb/cockroach/pkg/cli.Run+0x6c /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/cli/cli.go:141
# 0x159998c github.com/cockroachdb/cockroach/pkg/cli.Main+0x18c /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/cli/cli.go:50
# 0x15d759f main.main+0x1f /home/radu/go/src/github.com/cockroachdb/cockroach/main.go:26
# 0x5769d9 runtime.main+0x209 /home/radu/go1.8/src/runtime/proc.go:185
1 @ 0x576e2a 0x586234 0x584e9c 0x5a10ad 0x5a44e1
# 0x576e29 runtime.gopark+0x139 /home/radu/go1.8/src/runtime/proc.go:271
# 0x586233 runtime.selectgoImpl+0x1363 /home/radu/go1.8/src/runtime/select.go:423
# 0x584e9b runtime.selectgo+0x1b /home/radu/go1.8/src/runtime/select.go:238
# 0x5a10ac runtime.ensureSigM.func1+0x2dc /home/radu/go1.8/src/runtime/signal_unix.go:434
1 @ 0x576e2a 0x586234 0x584e9c 0x89523c 0x5a44e1
# 0x89523b net/http.(*persistConn).readLoop+0x9eb /home/radu/go1.8/src/net/http/transport.go:1599
1 @ 0x576e2a 0x586234 0x584e9c 0x89663a 0x5a44e1
# 0x896639 net/http.(*persistConn).writeLoop+0x439 /home/radu/go1.8/src/net/http/transport.go:1704
1 @ 0x576e2a 0x586234 0x584e9c 0x9d9f15 0x9dae25 0x5bb739 0x5bb8a8 0x9dad78 0x9ecf15 0x9edbfd 0x9f9622 0xd6f0b2 0xd80f15 0xd81070 0xc953a7 0x5a44e1
# 0x9d9f14 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read+0x494 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:126
# 0x9dae24 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read+0x54 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:391
# 0x5bb738 io.ReadAtLeast+0xa8 /home/radu/go1.8/src/io/io.go:307
# 0x5bb8a7 io.ReadFull+0x57 /home/radu/go1.8/src/io/io.go:325
# 0x9dad77 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read+0xc7 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:375
# 0x9ecf14 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg+0x64 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:265
# 0x9edbfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv+0x4c /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:361
# 0x9f9621 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg+0xf1 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:398
# 0xd6f0b1 github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv+0x61 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:194
# 0xd80f14 github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1+0x34 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:311
# 0xd8106f github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2+0xbf /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:319
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0x9e9d54 0x5a44e1
# 0x9e9d53 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor+0x693 /home/radu/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:927
1 @ 0x576e2a 0x586234 0x584e9c 0xd66272 0xd80ac0 0xc953a7 0x5a44e1
# 0xd66271 github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip+0x3e1 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:323
# 0xd80abf github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1+0x40f /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:130
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0xd82065 0xc953a7 0x5a44e1
# 0xd82064 github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1+0x454 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1093
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0xd82855 0xc953a7 0x5a44e1
# 0xd82854 github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1+0x324 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1122
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x576e2a 0x586234 0x584e9c 0xfeebf1 0x102b850 0xff3806 0x102bc79 0xc953a7 0x5a44e1
# 0xfeebf0 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess+0x240 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:208
# 0x102b84f github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1+0x8f /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:270
# 0xff3805 github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit+0x1e5 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:305
# 0x102bc78 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1+0x238 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:272
# 0xc953a6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1+0xf6 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198
1 @ 0x5a44e1
1 @ 0xa69b62 0xa69960 0xa65f65 0x14d0411 0x14d0723 0x8819c4 0x15360e9 0x8819c4 0x155520c 0x8819c4 0x882e00 0x1547a3d 0x8841c2 0x880532 0x5a44e1
# 0xa69b61 runtime/pprof.writeRuntimeProfile+0xa1 /home/radu/go1.8/src/runtime/pprof/pprof.go:632
# 0xa6995f runtime/pprof.writeGoroutine+0x9f /home/radu/go1.8/src/runtime/pprof/pprof.go:594
# 0xa65f64 runtime/pprof.(*Profile).WriteTo+0x3b4 /home/radu/go1.8/src/runtime/pprof/pprof.go:302
# 0x14d0410 net/http/pprof.handler.ServeHTTP+0x1d0 /home/radu/go1.8/src/net/http/pprof/pprof.go:209
# 0x14d0722 net/http/pprof.Index+0x1e2 /home/radu/go1.8/src/net/http/pprof/pprof.go:221
# 0x8819c3 net/http.HandlerFunc.ServeHTTP+0x43 /home/radu/go1.8/src/net/http/server.go:1942
# 0x15360e8 github.com/cockroachdb/cockroach/pkg/server.handleDebug+0x68 /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/debug.go:89
# 0x8819c3 net/http.HandlerFunc.ServeHTTP+0x43 /home/radu/go1.8/src/net/http/server.go:1942
# 0x155520b github.com/cockroachdb/cockroach/pkg/server.authorizedHandler.func1+0x7b /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/debug.go:80
# 0x8819c3 net/http.HandlerFunc.ServeHTTP+0x43 /home/radu/go1.8/src/net/http/server.go:1942
# 0x882dff net/http.(*ServeMux).ServeHTTP+0x12f /home/radu/go1.8/src/net/http/server.go:2238
# 0x1547a3c github.com/cockroachdb/cockroach/pkg/server.(*Server).ServeHTTP+0x11c /home/radu/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1198
# 0x8841c1 net/http.serverHandler.ServeHTTP+0x91 /home/radu/go1.8/src/net/http/server.go:2568
# 0x880531 net/http.(*conn).serve+0x611 /home/radu/go1.8/src/net/http/server.go:1825
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment