Skip to content

Instantly share code, notes, and snippets.

@tbg
Created June 23, 2021 09:35
Show Gist options
  • Save tbg/759603dad82172a76c224c379599e567 to your computer and use it in GitHub Desktop.
Save tbg/759603dad82172a76c224c379599e567 to your computer and use it in GitHub Desktop.
This file has been truncated, but you can view the full file.
goroutine 218101407 [running]:
github.com/cockroachdb/cockroach/pkg/server.(*statusServer).Stacks(0xc00589e840, 0x5a4e4e0, 0xc0060ee150, 0xc0133c2f60, 0xc00589e840, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/status.go:1196 +0x185
github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_Stacks_Handler.func1(0x5a4e4e0, 0xc0060ee0c0, 0x4a19660, 0xc0133c2f60, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:4779 +0x8b
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0060ee0c0, 0x4a19660, 0xc0133c2f60, 0xc0133c2f80, 0xc0133c2fa0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0060ee0c0, 0x4a19660, 0xc0133c2f60, 0xc0020f2990, 0xca72c8, 0x48dd840, 0xc016d27a40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0060ee0c0, 0x4a19660, 0xc0133c2f60, 0xc0133c2f80, 0xc016d27a40, 0xc016d27a40, 0x4, 0xc01aa324e0, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0060ee0c0, 0x4a19660, 0xc0133c2f60, 0x0, 0xc000600000, 0x0, 0xc006e800c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0060ee0c0, 0x4a19660, 0xc0133c2f60, 0xc0133c2f80, 0xc016d27a00, 0xc0020f2ac8, 0xca72c8, 0x48dd840, 0xc016d27a00)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0060ee0c0, 0x4a19660, 0xc0133c2f60, 0xc0133c2f80, 0xc0133c2fa0, 0xc0020f2ba0, 0x661046, 0x491f620, 0xc0060ee0c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_Stacks_Handler(0x4bcfa40, 0xc00589e840, 0x5a4e4e0, 0xc0060ee0c0, 0xc006e80060, 0xc001681820, 0x5a4e4e0, 0xc0060ee0c0, 0xc003a70000, 0x3)
/go/src/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:4781 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc00ded6780, 0xc014e5f700, 0xc00076a8a0, 0x7cd3c98, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc00ded6780, 0xc014e5f700, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0111e7660, 0xc0005601a0, 0x5aba220, 0xc00ded6780, 0xc014e5f700)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 1 [select]:
github.com/cockroachdb/cockroach/pkg/cli.runStart(0x7ccdd00, 0xc000a7b040, 0x0, 0x14, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/cli/start.go:853 +0xeab
github.com/cockroachdb/cockroach/pkg/cli.runStartJoin(0x7ccdd00, 0xc000a7b040, 0x0, 0x14, 0x4, 0x7f8b3be767d0)
/go/src/github.com/cockroachdb/cockroach/pkg/cli/start.go:264 +0x4e
github.com/cockroachdb/cockroach/pkg/cli.MaybeDecorateGRPCError.func1(0x7ccdd00, 0xc000a7b040, 0x0, 0x14, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/cli/error.go:194 +0x79
github.com/cockroachdb/cockroach/pkg/cli.maybeShoutError.func1(0x7ccdd00, 0xc000a7b040, 0x0, 0x14, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/cli/error.go:377 +0x4d
github.com/spf13/cobra.(*Command).execute(0x7ccdd00, 0xc000a7af00, 0x14, 0x14, 0x7ccdd00, 0xc000a7af00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:826 +0x47c
github.com/spf13/cobra.(*Command).ExecuteC(0x7cc3a80, 0xc00007c0a8, 0x4c0098a, 0x6)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:914 +0x30b
github.com/spf13/cobra.(*Command).Execute(...)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:864
github.com/cockroachdb/cockroach/pkg/cli.Run(...)
/go/src/github.com/cockroachdb/cockroach/pkg/cli/cli.go:322
github.com/cockroachdb/cockroach/pkg/cli.doMain(0x7ccdd00, 0xc000507bea, 0x5, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/cli/cli.go:121 +0x11a
github.com/cockroachdb/cockroach/pkg/cli.Main()
/go/src/github.com/cockroachdb/cockroach/pkg/cli/cli.go:59 +0x13a
main.main()
/go/src/github.com/cockroachdb/cockroach/pkg/cmd/cockroach/main.go:26 +0x25
goroutine 6 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x73
created by github.com/cockroachdb/cockroach/pkg/util/log.init.6
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x35
goroutine 7 [chan receive, 4360 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0x12c
created by github.com/cockroachdb/cockroach/pkg/util/log.init.6
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x4d
goroutine 34 [syscall]:
os/signal.signal_recv(0x59ffb60)
/usr/local/go/src/runtime/sigqueue.go:147 +0x9d
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x25
created by os/signal.Notify.func1.1
/usr/local/go/src/os/signal/signal.go:150 +0x45
goroutine 38 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc000bee370)
/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:154 +0x105
created by go.opencensus.io/stats/view.init.0
/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:32 +0x57
goroutine 50 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:91 +0x128
created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0
/go/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:79 +0x35
goroutine 41 [chan receive, 4360 minutes]:
github.com/cockroachdb/sentry-go.(*HTTPTransport).worker(0xc00166bb00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/sentry-go/transport.go:303 +0x7d
created by github.com/cockroachdb/sentry-go.(*HTTPTransport).Configure.func1
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/sentry-go/transport.go:174 +0x3e
goroutine 66 [chan receive, 4360 minutes]:
github.com/cockroachdb/cockroach/pkg/cli.runStart.func1(0xc000500960)
/go/src/github.com/cockroachdb/cockroach/pkg/cli/start.go:314 +0x34
created by github.com/cockroachdb/cockroach/pkg/cli.runStart
/go/src/github.com/cockroachdb/cockroach/pkg/cli/start.go:312 +0x16d1
goroutine 67 [select, 4360 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0002ba070, 0x5a4e420, 0xc0006ce180)
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0xbb
created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig
/go/src/github.com/cockroachdb/cockroach/pkg/util/log/flags.go:172 +0x1205
goroutine 69 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc00059af60)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc00059af60, 0xc0003abce0, 0x1, 0x1, 0xc00051ffb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0009e5700)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 70 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc00059b680)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc00059b680, 0xc0003abce0, 0x1, 0x1, 0xc000095fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000ad1a00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 71 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000cde000)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000cde000, 0xc0003abce0, 0x1, 0x1, 0xc0000abfb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d1e000)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 72 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc00059aed0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc00059aed0, 0xc0003abce0, 0x1, 0x1, 0xc000096fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d1e100)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 73 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000cde0c0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000cde0c0, 0xc0003abce0, 0x1, 0x1, 0xc0000977b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d1e200)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 74 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000f8c030)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000f8c030, 0xc0003abce0, 0x1, 0x1, 0xc000f96fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d1e300)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 75 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000cde060)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000cde060, 0xc0003abce0, 0x1, 0x1, 0xc000f53fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d1e400)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 76 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000cde120)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000cde120, 0xc0003abce0, 0x1, 0x1, 0xc000f547b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d1e500)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 83 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1(0xc00098e700)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go:63 +0x105
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go:58 +0x50
goroutine 44 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1(0xc003a76400)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go:63 +0x105
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go:58 +0x50
goroutine 98 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000d2b470)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000d2b470, 0xc0003abce0, 0x1, 0x1, 0xc000f4ffb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0058cc700)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 99 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000d2b4d0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000d2b4d0, 0xc0003abce0, 0x1, 0x1, 0xc000f507b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0058cc800)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 100 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000f8c090)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000f8c090, 0xc0003abce0, 0x1, 0x1, 0xc000f92fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0058cc900)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 101 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000d2b350)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000d2b350, 0xc0003abce0, 0x1, 0x1, 0xc000f517b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0058cca00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 102 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000d2b2f0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000d2b2f0, 0xc0003abce0, 0x1, 0x1, 0xc000f51fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0058ccb00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 103 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000d2b3b0)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000d2b3b0, 0xc0003abce0, 0x1, 0x1, 0xc000f52fb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0058ccc00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 104 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc000d2b410)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc000d2b410, 0xc0003abce0, 0x1, 0x1, 0xc000f537b8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0058ccd00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 105 [chan receive, 4360 minutes]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1(0x5a4e4e0, 0xc00140e060)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:176 +0x76
runtime/pprof.Do(0x5a4e4e0, 0xc00140e060, 0xc0003abce0, 0x1, 0x1, 0xc00187efb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0058cce00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:175 +0x89
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/table_cache.go:167 +0x1a5
goroutine 106 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1(0xc000ce5640)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go:63 +0x105
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go:58 +0x50
goroutine 84 [sync.Cond.Wait, 4360 minutes]:
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:312
sync.runtime_notifyListWait(0xc00032eae0, 0x0)
/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00032ead0)
/usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cockroachdb/pebble/internal/record.(*flusherCond).Wait(...)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/internal/record/log_writer.go:193
github.com/cockroachdb/pebble/internal/record.(*LogWriter).flushLoop(0xc00032ea00, 0x5a4e4e0, 0xc0032e8030)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/internal/record/log_writer.go:396 +0x10b
runtime/pprof.Do(0x5a4e4e0, 0xc0032e8030, 0xc0003ab3a0, 0x1, 0x1, 0xc00187afb8)
/usr/local/go/src/runtime/pprof/runtime.go:40 +0xcc
github.com/cockroachdb/pebble/internal/record.NewLogWriter.func2(0xc00032ea00)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/internal/record/log_writer.go:314 +0x89
created by github.com/cockroachdb/pebble/internal/record.NewLogWriter
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/internal/record/log_writer.go:313 +0x2ea
goroutine 119 [select, 4360 minutes]:
github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1(0x52fffa8, 0xc0005b88c0, 0xc002aa8930, 0x5a4e4e0, 0xc0018e6b10, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:230 +0x12d
created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps
/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:223 +0xd0
goroutine 216089778 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc019d56960, 0x1688bcc21f00e52d, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc019d56960, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc019d56930, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc019d56930, 0x4c245c5, 0x10, 0xc011dbb6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc019d56930, 0xc0119e2780, 0xc019d56930, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc019d568d0, 0xc0119e2780, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc019d568d0, 0x4bc7440, 0xc0119e2780, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc019d568d0, 0x4bc7440, 0xc0119e2780, 0xc00f6990e0, 0xc00f699100, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc019d568d0, 0x4bc7440, 0xc0119e2780, 0xc011dbb990, 0xca72c8, 0x48dd840, 0xc023047c40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc019d568d0, 0x4bc7440, 0xc0119e2780, 0xc00f6990e0, 0xc023047c40, 0xc023047c40, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc019d568d0, 0x4bc7440, 0xc0119e2780, 0x0, 0xc024c79800, 0x0, 0xc0212a67e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc019d568d0, 0x4bc7440, 0xc0119e2780, 0xc00f6990e0, 0xc023047b80, 0xc011dbbac8, 0xca72c8, 0x48dd840, 0xc023047b80)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc019d568d0, 0x4bc7440, 0xc0119e2780, 0xc00f6990e0, 0xc00f699100, 0xc01e93fba0, 0x661046, 0x491f620, 0xc019d568d0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc019d568d0, 0xc0212a6780, 0xc001681820, 0x5a4e4e0, 0xc019d568d0, 0xc011b8b200, 0x32b)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0117b6300, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0117b6300, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0117b6300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 111 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00265a140, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x125
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001b0fbc0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1d3
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002230780)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xd7
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10d6
goroutine 112 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002230780)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x28a
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10fb
goroutine 113 [IO wait]:
internal/poll.runtime_pollWait(0x7f8b125cb770, 0x72, 0x59b54a0)
/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc002ea7f18, 0x72, 0x59b5400, 0x7cf1b20, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002ea7f00, 0xc003aae000, 0x1029d, 0x1029d, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:159 +0x1a5
net.(*netFD).Read(0xc002ea7f00, 0xc003aae000, 0x1029d, 0x1029d, 0xc, 0xc, 0xc003aae005)
/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc002b9a560, 0xc003aae000, 0x1029d, 0x1029d, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:182 +0x8e
github.com/cockroachdb/cmux.(*MuxConn).Read(0xc002ecafc0, 0xc003aae000, 0x1029d, 0x1029d, 0xc003aae000, 0x5, 0x7f8b3bd0fbe8)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x115
crypto/tls.(*atLeastReader).Read(0xc0052e56e0, 0xc003aae000, 0x1029d, 0x1029d, 0x1e, 0x10298, 0xc001908798)
/usr/local/go/src/crypto/tls/conn.go:779 +0x62
bytes.(*Buffer).ReadFrom(0xc0012e2d00, 0x59a7c60, 0xc0052e56e0, 0x45f8c5, 0x458c0a0, 0x49151a0)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc0012e2a80, 0x7f8b12553238, 0xc002ecafc0, 0x5, 0xc002ecafc0, 0xd)
/usr/local/go/src/crypto/tls/conn.go:801 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc0012e2a80, 0x0, 0x0, 0xc3213e)
/usr/local/go/src/crypto/tls/conn.go:608 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:576
crypto/tls.(*Conn).Read(0xc0012e2a80, 0xc0022f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1252 +0x15f
bufio.(*Reader).Read(0xc001b0fb60, 0xc00106ae38, 0x9, 0x9, 0xc002730ae0, 0xc012c71320, 0xc001908dff)
/usr/local/go/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x59a7a20, 0xc001b0fb60, 0xc00106ae38, 0x9, 0x9, 0x9, 0xc001908dd0, 0x45ae85, 0xc002730ae0)
/usr/local/go/src/io/io.go:314 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:333
golang.org/x/net/http2.readFrameHeader(0xc00106ae38, 0x9, 0x9, 0x59a7a20, 0xc001b0fb60, 0x0, 0xc000000000, 0xc00263e2b0, 0x44e0040)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00106ae00, 0xc020d8eb50, 0x8702740, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002230780, 0xc00280ce10, 0x52fddb8)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0005601a0, 0x5aba220, 0xc002230780)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005601a0, 0x5aba220, 0xc002230780)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:702 +0x50b
goroutine 274 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595f8, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).Lock(0xc019f595f0)
/usr/local/go/src/sync/rwmutex.go:103 +0x85
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).updateProposalQuotaRaftMuLocked(0xc019f59200, 0x5a4e4e0, 0xc01fe6e180, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_proposal_quota.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(0xc019f59200, 0x5a4e4e0, 0xc01fe6e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_raft.go:884 +0xe5d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(0xc019f59200, 0x5a4e4e0, 0xc01fe6e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_raft.go:466 +0x113
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0018ad000, 0x5a4e4e0, 0xc00140e300, 0x5c79)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:523 +0x134
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0006fe3c0, 0x5a4e4e0, 0xc00140e300)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:284 +0x2c2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc00140e300, 0xc000a70b40, 0xc002f922d0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 303 [semacquire, 4360 minutes]:
sync.runtime_Semacquire(0xc0006fe450)
/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc0006fe448)
/usr/local/go/src/sync/waitgroup.go:130 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait(...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:212
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc00140e300, 0xc000a70b40, 0xc002f924a0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 306 [semacquire, 86 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0190ebb00, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc0190ebb30, 0xc0072a1e58)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:372 +0x1db
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).systemGossipUpdate(0xc0018ad000, 0xc01a9b4eb0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1926 +0x1db
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func2(0x5a4e4e0, 0xc00140e300)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1547 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc00140e300, 0xc000a70b40, 0xc002f84b20)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 315 [semacquire, 187 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc00399f000, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).OwnsValidLease(0xc019f59200, 0x5a4e4e0, 0xc002f83350, 0x1688bd08ab7043ac, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_range_lease.go:688 +0xee
github.com/cockroachdb/cockroach/pkg/kv/kvserver.shouldNotMoveAway(0x5a4e4e0, 0xc002f83350, 0xc019f59200, 0x4039c61e08039b6f, 0xc006c60300, 0x1688bd08ab7043ac, 0x0, 0x40696dc390f84fc4, 0x40696dc390f84fc4)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_rebalancer.go:901 +0x7e
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).chooseLeaseToTransfer(0xc00292bd40, 0x5a4e4e0, 0xc002f83350, 0xc00399fa78, 0xc006c60300, 0xc006c60000, 0x3, 0x4, 0x4008000000000000, 0x40a1c60000000000, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_rebalancer.go:390 +0x1ce
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).rebalanceStore(0xc00292bd40, 0x5a4e4e0, 0xc002f83350, 0x2, 0xc006c60000, 0x3, 0x4, 0x4008000000000000, 0x40a1c60000000000, 0x4008000000000000, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_rebalancer.go:246 +0x445
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1(0x5a4e4e0, 0xc002f83350)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_rebalancer.go:204 +0x378
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc002f83350, 0xc000a70b40, 0xc002f84b80)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 325 [semacquire, 186 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc004b51c00, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc004b51ce0, 0xc01f79e000)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:372 +0x1db
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1(0x5a4e4e0, 0xc00140e330)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:277 +0x1b3
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc00140e330, 0x0, 0xc002efe4c0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 316 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc005810b00, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc005810b40, 0xc01e0657a8)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:372 +0x1db
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Capacity(0xc0018ad000, 0x5a4e4e0, 0xc0058105a0, 0x10100c0e7695000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2385 +0x4ed
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Descriptor(0xc0018ad000, 0x5a4e4e0, 0xc0058105a0, 0x5a4e400, 0xc000c14e70, 0x5a4e4e0, 0xc0058105a0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2455 +0x69
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).updateCapacityGauges(0xc0018ad000, 0x5a4e4e0, 0xc0058105a0, 0x5a4e4e0, 0xc0058105a0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:796 +0x48
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).ComputeMetrics(0xc0018ad000, 0x5a4e4e0, 0xc000c14e70, 0x61ae, 0x0, 0x5a28de0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2636 +0x9a
github.com/cockroachdb/cockroach/pkg/server.(*Node).computePeriodicMetrics.func1(0xc0018ad000, 0x464925, 0xc01e93dd48)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:676 +0x57
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores.func1(0x1, 0xc0018ad000, 0xc01e93dd48)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:148 +0x38
github.com/cockroachdb/cockroach/pkg/util/syncutil.(*IntMap).Range(0xc0002f7c30, 0xc01e065dd8)
/go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/int_map.go:352 +0x130
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores(0xc0002f7c00, 0xc01e93de20, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:147 +0x75
github.com/cockroachdb/cockroach/pkg/server.(*Node).computePeriodicMetrics(0xc0058cac00, 0x5a4e4e0, 0xc000c14e70, 0x61ae, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:675 +0x77
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x5a4e4e0, 0xc000c14e70)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:662 +0x165
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0x0, 0xc002f84f80)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 317 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc008b4d900, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc008b4d9b0, 0xc00cf9b8d8)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:372 +0x1db
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Capacity(0xc0018ad000, 0x5a4e4e0, 0xc0018e6b10, 0xc003c1d400, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2385 +0x4ed
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Descriptor(0xc0018ad000, 0x5a4e4e0, 0xc0018e6b10, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2455 +0x69
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).GossipStore(0xc0018ad000, 0x5a4e4e0, 0xc0018e6b10, 0x450e700, 0xc003b1c908, 0xc002f833b0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1967 +0x78
github.com/cockroachdb/cockroach/pkg/server.(*Node).gossipStores.func1(0xc0018ad000, 0x464925, 0xc021f63c88)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:644 +0x45
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores.func1(0x1, 0xc0018ad000, 0xc021f63c88)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:148 +0x38
github.com/cockroachdb/cockroach/pkg/util/syncutil.(*IntMap).Range(0xc0002f7c30, 0xc00cf9bd18)
/go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/int_map.go:352 +0x130
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores(0xc0002f7c00, 0xc021f63d98, 0x44b0be0, 0xc0116e13e0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:147 +0x75
github.com/cockroachdb/cockroach/pkg/server.(*Node).gossipStores(0xc0058cac00, 0x5a4e4e0, 0xc0018e6b10)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:643 +0x7a
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossiping.func1(0x5a4e4e0, 0xc0018e6b10)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:629 +0x3b6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc0018e6b10, 0xc000a70b40, 0xc002f84fa0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 218080135 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc022efd050, 0x1688c728044f5742, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc022efd050, 0x1688c728044f5742, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc022efd020, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc022efd020, 0x4c245c5, 0x10, 0xc01f8b16c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc022efd020, 0xc01f6c9c20, 0xc022efd020, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc022efcfc0, 0xc01f6c9c20, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc022efcfc0, 0x4bc7440, 0xc01f6c9c20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc022efcfc0, 0x4bc7440, 0xc01f6c9c20, 0xc019077f20, 0xc019077f40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc022efcfc0, 0x4bc7440, 0xc01f6c9c20, 0xc01f8b1990, 0xca72c8, 0x48dd840, 0xc005397f00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc022efcfc0, 0x4bc7440, 0xc01f6c9c20, 0xc019077f20, 0xc005397f00, 0xc005397f00, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc022efcfc0, 0x4bc7440, 0xc01f6c9c20, 0x0, 0xc000600000, 0x0, 0xc00708fb60)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc022efcfc0, 0x4bc7440, 0xc01f6c9c20, 0xc019077f20, 0xc005397ec0, 0xc01f8b1ac8, 0xca72c8, 0x48dd840, 0xc005397ec0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc022efcfc0, 0x4bc7440, 0xc01f6c9c20, 0xc019077f20, 0xc019077f40, 0xc002f74ba0, 0x661046, 0x491f620, 0xc022efcfc0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc022efcfc0, 0xc00708fb00, 0xc001681820, 0x5a4e4e0, 0xc022efcfc0, 0xc01f0ca000, 0x429)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc028f59400, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc028f59400, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc028f59400)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217757651 [semacquire, 31 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01fcd6990, 0x1688c58a14900b9d, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01fcd6990, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01fcd6990, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01fcd6990, 0x4c245c5, 0x10, 0xc0073716c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01fcd6990, 0xc01f132c80, 0xc01fcd6990, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a6dfa0, 0xc00511d018, 0xc01f132c80, 0xc0058cac00, 0xc01fcd6930, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a6dfa0, 0xc00511d018, 0x4bc7440, 0xc01f132c80, 0x21, 0xc007371890, 0x2, 0x2)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01fcd68d0, 0x4bc7440, 0xc01f132c80, 0xc010f84100, 0xc010f84120, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:137 +0x2dc
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01fcd68d0, 0x4bc7440, 0xc01f132c80, 0xc007371990, 0xca72c8, 0x48dd840, 0xc00e7b2740)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01fcd68d0, 0x4bc7440, 0xc01f132c80, 0xc010f84100, 0xc00e7b2740, 0xc00e7b2740, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01fcd68d0, 0x4bc7440, 0xc01f132c80, 0x0, 0xc003d76000, 0x0, 0xc007322f60)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01fcd68d0, 0x4bc7440, 0xc01f132c80, 0xc010f84100, 0xc00e7b2700, 0xc007371ac8, 0xca72c8, 0x48dd840, 0xc00e7b2700)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01fcd68d0, 0x4bc7440, 0xc01f132c80, 0xc010f84100, 0xc010f84120, 0xc002f7bba0, 0x661046, 0x491f620, 0xc01fcd68d0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01fcd68d0, 0xc007322f00, 0xc001681820, 0x5a4e4e0, 0xc01fcd68d0, 0xc00bf89800, 0x348)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00d43b900, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00d43b900, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00d43b900)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217758702 [semacquire, 31 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc001f88e78, 0x1688c58b3581d86a, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc001f88e78, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc019e7f170, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc019e7f170, 0x4c245c5, 0x10, 0xc0292803f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc019e7f170, 0xc013c97400, 0xc019e7f170, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc019e7f140, 0xc013c97400, 0xc001f88b38, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc010728f80, 0x5a4e4e0, 0xc019e7f140, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc010728f80, 0x5a6dfa0, 0xc001f88b38, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc001f88b38, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc001f88b38, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc001f88b38, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc001f88b38, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc001f88658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc002d16d50, 0x5a6dfa0, 0xc001f88658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc002d16d18, 0x5a6dfa0, 0xc001f88658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc002d16ce8, 0x5a6dfa0, 0xc001f88658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc002d16c48, 0x5a6dfa0, 0xc001f88658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc002d16c48, 0x5a6dfa0, 0xc001f88658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc002d16b78, 0x5a6dfa0, 0xc001f88658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc002d16b58, 0x5a6dfa0, 0xc001f88658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc002d16ac0, 0x5a6dfa0, 0xc001f88658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc002d16900, 0x5a4e420, 0xc010402bc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc010402bc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc0203d3cb0, 0x5a4e420, 0xc010402bc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc010402bc0, 0xc0292851f0, 0xc00df69900, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc0203d3cb0, 0x5a4e420, 0xc010402bc0, 0xc00df69900, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc0203d3cb0, 0x5a4e420, 0xc010402bc0, 0xc00df69900, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc021a70c28, 0x5a4e420, 0xc010402bc0, 0xc003b4fb00, 0xc016618ae0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc021a70c00, 0x5a4e420, 0xc010402bc0, 0xc003b4fb00, 0xc016618ae0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc00a6545c0, 0x5a4e420, 0xc010402bc0, 0xc003b4fb00, 0xc016618ae0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc00a6545c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc029285588, 0x4bfc8f0, 0x5, 0x5a50860, 0xc00a6545c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc015525588, 0x5a50860, 0xc00a6545c0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc015525588, 0x5a50860, 0xc00a6545c0, 0xc, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc010402bc0, 0xc003b4fb00, 0xc016618ae0, 0x5a50860, 0xc00a6545c0, 0x5a4e420, 0xc010402bc0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc00f273200, 0x5a4e420, 0xc010402bc0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc00890b200)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc015525768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc0125d8000, 0x5a4e420, 0xc010402bc0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc0125d8000, 0x5a4e420, 0xc010402bc0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc01fd086c0, 0x5a4e420, 0xc010402bc0, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc0068c4ee0, 0xc0203d3cb0, 0xc00f20ad00, 0xc014b5f500, 0xc016618cf8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc002160318, 0xc016618cf8, 0xc0068c4ee0, 0xc0203d3cb0, 0x5a50860, 0xc00a6545c0, 0x0, 0xc014b5f500, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc016618600, 0x5a6dfa0, 0xc002160318, 0xc016618ae0, 0x2, 0x7f8b0dd7dcb0, 0xc009a4a180, 0xc015526000, 0xc01ae36d18, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc016618600, 0x5a6dfa0, 0xc002160318, 0xc016618ae0, 0x7f8b0dd7dcb0, 0xc009a4a180, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc016618600, 0x5a6dfa0, 0xc002160318, 0x5a6dda0, 0xc00f20a380, 0xc009dfc030, 0x1e9, 0x11, 0x1, 0xc026c93950, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc016618600, 0x5a4e420, 0xc01549e000, 0x5a6dda0, 0xc00f20a380, 0xc009dfc030, 0x1e9, 0x11, 0x1, 0xc026c93950, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc016618600, 0x5a4e420, 0xc01549e000, 0xc026c93950, 0xc010709c20, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc009dfc39b, 0x0, 0x0, 0x153585fd, 0xed85aa683, 0x0, 0xc016618600, 0xc029287a20, 0xc029287a10, 0x5, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc016618600, 0x5a4e420, 0xc01549e000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc016618600, 0x5a4e420, 0xc02338d840, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc004e0c620, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc02338d840, 0xc016618600, 0x5400, 0x15000, 0xc001805a40, 0xc004e0c620, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc003094426, 0xc00ca34940, 0x5a4e420, 0xc02338d840, 0xc004e0c620, 0xc001a38000, 0xc009a49b00, 0x5ab5540, 0xc00cabcc80, 0xc0052d9f80, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 218079881 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc00db39ce0, 0x1688c727f8d80826, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc00db39ce0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00db39cb0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00db39cb0, 0x4c245c5, 0x10, 0xc01e3776c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00db39cb0, 0xc01b0d74a0, 0xc00db39cb0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00db39c50, 0xc01b0d74a0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc00db39c50, 0x4bc7440, 0xc01b0d74a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc00db39c50, 0x4bc7440, 0xc01b0d74a0, 0xc021e53f40, 0xc021e53f60, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00db39c50, 0x4bc7440, 0xc01b0d74a0, 0xc01e377990, 0xca72c8, 0x48dd840, 0xc01d5d3800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc00db39c50, 0x4bc7440, 0xc01b0d74a0, 0xc021e53f40, 0xc01d5d3800, 0xc01d5d3800, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00db39c50, 0x4bc7440, 0xc01b0d74a0, 0x0, 0xc000100800, 0x0, 0xc00e9210e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc00db39c50, 0x4bc7440, 0xc01b0d74a0, 0xc021e53f40, 0xc01d5d37c0, 0xc01e377ac8, 0xca72c8, 0x48dd840, 0xc01d5d37c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc00db39c50, 0x4bc7440, 0xc01b0d74a0, 0xc021e53f40, 0xc021e53f60, 0xc003019ba0, 0x661046, 0x491f620, 0xc00db39c50)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc00db39c50, 0xc00e921080, 0xc001681820, 0x5a4e4e0, 0xc00db39c50, 0xc01dfc8000, 0x284a)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02984e600, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02984e600, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02984e600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216101003 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01a6619e0, 0x1688bcc8c90b9962, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01a6619e0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01a6619b0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01a6619b0, 0x4c245c5, 0x10, 0xc0032676c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01a6619b0, 0xc015235540, 0xc01a6619b0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01a661950, 0xc015235540, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01a661950, 0x4bc7440, 0xc015235540, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01a661950, 0x4bc7440, 0xc015235540, 0xc019d72960, 0xc019d72980, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01a661950, 0x4bc7440, 0xc015235540, 0xc003267990, 0xca72c8, 0x48dd840, 0xc011370d80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01a661950, 0x4bc7440, 0xc015235540, 0xc019d72960, 0xc011370d80, 0xc011370d80, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01a661950, 0x4bc7440, 0xc015235540, 0x0, 0xc00758a000, 0x0, 0xc020796000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01a661950, 0x4bc7440, 0xc015235540, 0xc019d72960, 0xc011370d00, 0xc003267ac8, 0xca72c8, 0x48dd840, 0xc011370d00)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01a661950, 0x4bc7440, 0xc015235540, 0xc019d72960, 0xc019d72980, 0xc001b0bba0, 0x661046, 0x491f620, 0xc01a661950)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01a661950, 0xc002ae9c20, 0xc001681820, 0x5a4e4e0, 0xc01a661950, 0xc020fdac00, 0x368)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0133d3300, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0133d3300, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0133d3300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217807283 [semacquire, 27 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc020764810, 0x1688c5c3c0af3d71, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc020764810, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0207647b0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0207647b0, 0x4c245c5, 0x10, 0xc00892b6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0207647b0, 0xc01b307cc0, 0xc0207647b0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc020764750, 0xc01b307cc0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc020764750, 0x4bc7440, 0xc01b307cc0, 0x0, 0x0, 0x0, 0x7cc94)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc020764750, 0x4bc7440, 0xc01b307cc0, 0xc01ed45060, 0xc01ed45080, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc020764750, 0x4bc7440, 0xc01b307cc0, 0x4c4c07, 0xca734a, 0x48dd840, 0xc0198f4100)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc020764750, 0x4bc7440, 0xc01b307cc0, 0xc01ed45060, 0xc0198f4100, 0xc0198f4100, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc020764750, 0x4bc7440, 0xc01b307cc0, 0x0, 0xc0002a9800, 0xc004bd9200, 0xc00892bb30)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc020764750, 0x4bc7440, 0xc01b307cc0, 0xc01ed45060, 0xc006802500, 0x4c4c07, 0xca734a, 0x48dd840, 0xc006802500)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc020764750, 0x4bc7440, 0xc01b307cc0, 0xc01ed45060, 0xc01ed45080, 0x4c4c07, 0x6610c5, 0x491f620, 0xc020764750)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc020764750, 0xc00f6901e0, 0xc001681820, 0x5a4e4e0, 0xc020764750, 0xc00673d200, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007cf5e00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007cf5e00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007cf5e00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218079952 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02b7f39e0, 0x1688c727f97c3a0a, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02b7f39e0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02b7f39b0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02b7f39b0, 0x4c245c5, 0x10, 0xc01e3976c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02b7f39b0, 0xc0298425a0, 0xc02b7f39b0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02b7f3950, 0xc0298425a0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02b7f3950, 0x4bc7440, 0xc0298425a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02b7f3950, 0x4bc7440, 0xc0298425a0, 0xc024016440, 0xc024016460, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02b7f3950, 0x4bc7440, 0xc0298425a0, 0xc01e397990, 0xca72c8, 0x48dd840, 0xc015206200)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02b7f3950, 0x4bc7440, 0xc0298425a0, 0xc024016440, 0xc015206200, 0xc015206200, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02b7f3950, 0x4bc7440, 0xc0298425a0, 0x0, 0xc00e8e4800, 0x0, 0xc0315bf140)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02b7f3950, 0x4bc7440, 0xc0298425a0, 0xc024016440, 0xc0152061c0, 0xc01e397ac8, 0xca72c8, 0x48dd840, 0xc0152061c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02b7f3950, 0x4bc7440, 0xc0298425a0, 0xc024016440, 0xc024016460, 0xc00301dba0, 0x661046, 0x491f620, 0xc02b7f3950)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02b7f3950, 0xc0315bf0e0, 0xc001681820, 0x5a4e4e0, 0xc02b7f3950, 0xc01e260000, 0x330b)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01df61b00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01df61b00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01df61b00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217756263 [chan receive]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00bae3600, 0x5a4e4e0, 0xc013ad81e0, 0xc00d38eb90, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc013ad81e0, 0x5ab2120, 0xc010eeaa80, 0xc004aff028, 0xd, 0xc013ad81b0, 0x59fb820, 0xc01a452cc0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc013ad81e0, 0x5ab21e0, 0xc02446d740, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc02446d740)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc02446d740, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 6651 [semacquire, 160 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc01d6efb00, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc01d6efb60, 0xc003ba2e30)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:372 +0x1db
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Capacity(0xc0018ad000, 0x5a4e4e0, 0xc0054649c0, 0xb2e81700, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2385 +0x4ed
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Descriptor(0xc0018ad000, 0x5a4e4e0, 0xc0054649c0, 0xc000c43500, 0x0, 0x70, 0x7f8b3be789b8)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2455 +0x69
github.com/cockroachdb/cockroach/pkg/server/status.(*MetricsRecorder).GenerateNodeStatus(0xc000602f00, 0x5a4e4e0, 0xc0054649c0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/status/recorder.go:467 +0x7a9
github.com/cockroachdb/cockroach/pkg/server/diagnostics.(*Reporter).populateNodeInfo(0xc0018dc630, 0x5a4e4e0, 0xc0054649c0, 0x3d850, 0xc01fd0a000)
/go/src/github.com/cockroachdb/cockroach/pkg/server/diagnostics/reporter.go:272 +0x65
github.com/cockroachdb/cockroach/pkg/server/diagnostics.(*Reporter).CreateReport(0xc0018dc630, 0x5a4e4e0, 0xc0054649c0, 0x4c12201, 0xb)
/go/src/github.com/cockroachdb/cockroach/pkg/server/diagnostics/reporter.go:175 +0xf1d
github.com/cockroachdb/cockroach/pkg/server/diagnostics.(*Reporter).ReportDiagnostics(0xc0018dc630, 0x5a4e4e0, 0xc000f83b90)
/go/src/github.com/cockroachdb/cockroach/pkg/server/diagnostics/reporter.go:122 +0xe6
github.com/cockroachdb/cockroach/pkg/server/diagnostics.(*Reporter).PeriodicallyReportDiagnostics.func1(0x5a4e4e0, 0xc000f83b90)
/go/src/github.com/cockroachdb/cockroach/pkg/server/diagnostics/reporter.go:98 +0x3d1
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc000f83b90, 0xc000a70b40, 0xc009daee80)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 4881 [IO wait]:
internal/poll.runtime_pollWait(0x7f8b125cb118, 0x72, 0x59b54a0)
/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc002657f18, 0x72, 0x59b5400, 0x7cf1b20, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002657f00, 0xc00998c000, 0x6ca6, 0x6ca6, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:159 +0x1a5
net.(*netFD).Read(0xc002657f00, 0xc00998c000, 0x6ca6, 0x6ca6, 0xc, 0xc, 0xc00998c005)
/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc0005a4810, 0xc00998c000, 0x6ca6, 0x6ca6, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:182 +0x8e
github.com/cockroachdb/cmux.(*MuxConn).Read(0xc003780fc0, 0xc00998c000, 0x6ca6, 0x6ca6, 0xc00998c000, 0x5, 0x7f8a80f8eee8)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x115
crypto/tls.(*atLeastReader).Read(0xc0133c3020, 0xc00998c000, 0x6ca6, 0x6ca6, 0x1e, 0x6ca1, 0xc003257798)
/usr/local/go/src/crypto/tls/conn.go:779 +0x62
bytes.(*Buffer).ReadFrom(0xc0035be280, 0x59a7c60, 0xc0133c3020, 0x45f8c5, 0x458c0a0, 0x49151a0)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc0035be000, 0x7f8b12553238, 0xc003780fc0, 0x5, 0xc003780fc0, 0xd)
/usr/local/go/src/crypto/tls/conn.go:801 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc0035be000, 0x0, 0x0, 0xc3213e)
/usr/local/go/src/crypto/tls/conn.go:608 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:576
crypto/tls.(*Conn).Read(0xc0035be000, 0xc0035e0000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1252 +0x15f
bufio.(*Reader).Read(0xc0035c1560, 0xc0011ce8f8, 0x9, 0x9, 0xc0035d0060, 0xc009da6300, 0xc003257dff)
/usr/local/go/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x59a7a20, 0xc0035c1560, 0xc0011ce8f8, 0x9, 0x9, 0x9, 0xc003257dd0, 0x45ae85, 0xc0035d0060)
/usr/local/go/src/io/io.go:314 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:333
golang.org/x/net/http2.readFrameHeader(0xc0011ce8f8, 0x9, 0x9, 0x59a7a20, 0xc0035c1560, 0x0, 0xc000000000, 0xc002e06e90, 0x44e0040)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011ce8c0, 0xc014442a40, 0x8702740, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0022b7e00, 0xc00210cb40, 0x52fddb8)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0005601a0, 0x5aba220, 0xc0022b7e00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005601a0, 0x5aba220, 0xc0022b7e00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:702 +0x50b
goroutine 216997058 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc021d1bce0, 0x1688c18bdf70598c, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc021d1bce0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc021d1bcb0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc021d1bcb0, 0x4c245c5, 0x10, 0xc004b696c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc021d1bcb0, 0xc01dff00a0, 0xc021d1bcb0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc021d1bc50, 0xc01dff00a0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc021d1bc50, 0x4bc7440, 0xc01dff00a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc021d1bc50, 0x4bc7440, 0xc01dff00a0, 0xc02309cf00, 0xc02309cf20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc021d1bc50, 0x4bc7440, 0xc01dff00a0, 0xc004b69990, 0xca72c8, 0x48dd840, 0xc00fe39640)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc021d1bc50, 0x4bc7440, 0xc01dff00a0, 0xc02309cf00, 0xc00fe39640, 0xc00fe39640, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc021d1bc50, 0x4bc7440, 0xc01dff00a0, 0x0, 0xc00fba5800, 0x0, 0xc01325a8a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc021d1bc50, 0x4bc7440, 0xc01dff00a0, 0xc02309cf00, 0xc00fe39600, 0xc004b69ac8, 0xca72c8, 0x48dd840, 0xc00fe39600)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc021d1bc50, 0x4bc7440, 0xc01dff00a0, 0xc02309cf00, 0xc02309cf20, 0xc003016ba0, 0x661046, 0x491f620, 0xc021d1bc50)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc021d1bc50, 0xc01325a7e0, 0xc001681820, 0x5a4e4e0, 0xc021d1bc50, 0xc021943200, 0x31a)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc013f2d700, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc013f2d700, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc013f2d700)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216091915 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0288a5860, 0x1688bcc45f78f389, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0288a5860, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0288a5860, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0288a5860, 0x4c245c5, 0x10, 0xc003a08410, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0288a5860, 0xc014acd4a0, 0xc0288a5860, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0288a5830, 0xc014acd4a0, 0xc016c37ec0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc01700ce00, 0x5a4e4e0, 0xc0288a5830, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc01700ce00, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc023f6cd50, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc023f6cd18, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc023f6cce8, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc023f6cc48, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc023f6cc48, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc023f6cb78, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc023f6cb58, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc023f6cac0, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc023f6c900, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00600bcb0, 0x5a4e420, 0xc016c37ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc016c37ec0, 0xc003a0d208, 0xc012866a00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc00600bcb0, 0x5a4e420, 0xc016c37ec0, 0xc012866a00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc00600bcb0, 0x5a4e420, 0xc016c37ec0, 0xc012866a00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc024ec6028, 0x5a4e420, 0xc016c37ec0, 0xc00d526600, 0xc00f2b50e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc024ec6000, 0x5a4e420, 0xc016c37ec0, 0xc00d526600, 0xc00f2b50e0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc02315bde0, 0x5a4e420, 0xc016c37ec0, 0xc00d526600, 0xc00f2b50e0, 0xc003a0d400, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc02315bde0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc003a0d5a0, 0x4bfc8f0, 0x5, 0x5a50860, 0xc02315bde0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc003a0d5a0, 0x5a50860, 0xc02315bde0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc003a0d5a0, 0x5a50860, 0xc02315bde0, 0x0, 0xc0239163c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc016c37ec0, 0xc00d526600, 0xc00f2b50e0, 0x5a50860, 0xc02315bde0, 0x5a4e420, 0xc016c37ec0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc012feec00, 0x5a4e420, 0xc016c37ec0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc0017da900)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc003a0d768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc00edc2600, 0x5a4e420, 0xc016c37ec0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc00edc2600, 0x5a4e420, 0xc016c37ec0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc006a6e360, 0x5a4e420, 0xc016c37ec0, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc019228620, 0xc00600bcb0, 0xc0133c0c00, 0xc01e1adc00, 0xc00f2b52f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc01f3033e0, 0xc00f2b52f8, 0xc019228620, 0xc00600bcb0, 0x5a50860, 0xc02315bde0, 0x0, 0xc01e1adc00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00f2b4c00, 0x5a4e4e0, 0xc01f3033e0, 0xc00f2b50e0, 0x2, 0x7f8b0dd7dcb0, 0xc0123b4680, 0xc003a0e000, 0xc019873cd8, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00f2b4c00, 0x5a4e4e0, 0xc01f3033e0, 0xc00f2b50e0, 0x7f8b0dd7dcb0, 0xc0123b4680, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00f2b4c00, 0x5a4e4e0, 0xc01f3033e0, 0x5a6dda0, 0xc022783880, 0xc026a38035, 0x1e9, 0x11, 0x1, 0xc0237f6ff0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00f2b4c00, 0x5a4e420, 0xc0166faac0, 0x5a6dda0, 0xc022783880, 0xc026a38035, 0x1e9, 0x11, 0x1, 0xc0237f6ff0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00f2b4c00, 0x5a4e420, 0xc0166faac0, 0xc0237f6ff0, 0xc011f77e60, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc005e04853, 0x0, 0x0, 0xfff1def, 0xed85a80d1, 0x0, 0xc00f2b4c00, 0xc003a0fa20, 0xc003a0fa10, 0x2a3, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00f2b4c00, 0x5a4e420, 0xc0166faac0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00f2b4c00, 0x5a4e420, 0xc02449a2c0, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc003afa6f0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc02449a2c0, 0xc00f2b4c00, 0x5400, 0x15000, 0xc001805a40, 0xc003afa6f0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc0215ac1f6, 0xc016fc6660, 0x5a4e420, 0xc02449a2c0, 0xc003afa6f0, 0xc001a38000, 0xc0123b4000, 0x5ab5540, 0xc024ec0200, 0xc00637eea0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 217635648 [semacquire, 47 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0x1688c4b39c562200, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Metrics(0xc019f59200, 0x5a4e4e0, 0xc007df5440, 0x1688c4b44d90e2ae, 0x0, 0xc007df54a0, 0x3, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_metrics.go:56 +0x55d
github.com/cockroachdb/cockroach/pkg/server.(*statusServer).rangesHelper.func4.1(0x5c79, 0xc00d399360, 0x49, 0x50, 0xc00c9cac78, 0x3, 0x8, 0xc01343a060, 0x3, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/server/status.go:1704 +0x152
github.com/cockroachdb/cockroach/pkg/kv/kvserver.IterateRangeDescriptors.func1(0xc012278fe2, 0x56, 0x701e, 0xc012279042, 0x82, 0x6fbe, 0x1688b2d94ad89416, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1341 +0x242
github.com/cockroachdb/cockroach/pkg/storage.MVCCIterate(0x5a4e4e0, 0xc007df5440, 0x5ab9f20, 0xc000c0e820, 0xc0195acdc0, 0xd, 0x10, 0xc008a2be00, 0xb, 0x10, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2658 +0x2da
github.com/cockroachdb/cockroach/pkg/kv/kvserver.IterateRangeDescriptors(0x5a4e4e0, 0xc007df5440, 0x5ab9f20, 0xc000c0e820, 0xc005d8f078, 0x873fed8, 0x203005)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1347 +0x2d6
github.com/cockroachdb/cockroach/pkg/server.(*statusServer).rangesHelper.func4(0xc0018ad000, 0x464925, 0xc005d8f558)
/go/src/github.com/cockroachdb/cockroach/pkg/server/status.go:1693 +0x53b
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores.func1(0x1, 0xc0018ad000, 0xc005d8f558)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:148 +0x38
github.com/cockroachdb/cockroach/pkg/util/syncutil.(*IntMap).Range(0xc0002f7c30, 0xc0066b35e8)
/go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/int_map.go:352 +0x130
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores(0xc0002f7c00, 0xc005d8f6e0, 0xc00092c000, 0xc024d64001)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:147 +0x75
github.com/cockroachdb/cockroach/pkg/server.(*statusServer).rangesHelper(0xc00589e840, 0x5a4e4e0, 0xc007df5440, 0xc007df53b0, 0x0, 0x0, 0x4af0580, 0x4bcfa40, 0xc0124e2300, 0x5baa840)
/go/src/github.com/cockroachdb/cockroach/pkg/server/status.go:1686 +0x35a
github.com/cockroachdb/cockroach/pkg/server.(*statusServer).Ranges(0xc00589e840, 0x5a4e4e0, 0xc007df5320, 0xc007df53b0, 0xc00589e840, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/status.go:1553 +0x51
github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_Ranges_Handler.func1(0x5a4e4e0, 0xc007df5320, 0x4a186a0, 0xc007df53b0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:4563 +0x8b
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc007df5320, 0x4a186a0, 0xc007df53b0, 0xc014cf6a80, 0xc014cf6aa0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc007df5320, 0x4a186a0, 0xc007df53b0, 0xc01e940990, 0xca72c8, 0x48dd840, 0xc01fd97fc0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc007df5320, 0x4a186a0, 0xc007df53b0, 0xc014cf6a80, 0xc01fd97fc0, 0xc01fd97fc0, 0x4, 0xc009a17530, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc007df5320, 0x4a186a0, 0xc007df53b0, 0x0, 0xc000600000, 0x0, 0xc01f1ff020)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc007df5320, 0x4a186a0, 0xc007df53b0, 0xc014cf6a80, 0xc01fd97ec0, 0xc01e940ac8, 0xca72c8, 0x48dd840, 0xc01fd97ec0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc007df5320, 0x4a186a0, 0xc007df53b0, 0xc014cf6a80, 0xc014cf6aa0, 0xc01e940ba0, 0x661046, 0x491f620, 0xc007df5320)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_Ranges_Handler(0x4bcfa40, 0xc00589e840, 0x5a4e4e0, 0xc007df5320, 0xc01f1fefc0, 0xc001681820, 0x5a4e4e0, 0xc007df5320, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:4565 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc009a50180, 0xc01466ed00, 0xc00076a8a0, 0x7cd3b78, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc009a50180, 0xc01466ed00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc010a6fbe0, 0xc0005601a0, 0x5aba220, 0xc009a50180, 0xc01466ed00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216086305 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01f4866c0, 0x1688bcc07b7268de, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01f4866c0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01f486690, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01f486690, 0x4c245c5, 0x10, 0xc011b696c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01f486690, 0xc002dfe1e0, 0xc01f486690, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01f486630, 0xc002dfe1e0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01f486630, 0x4bc7440, 0xc002dfe1e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01f486630, 0x4bc7440, 0xc002dfe1e0, 0xc01fc937a0, 0xc01fc937c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01f486630, 0x4bc7440, 0xc002dfe1e0, 0xc011b69990, 0xca72c8, 0x48dd840, 0xc023653400)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01f486630, 0x4bc7440, 0xc002dfe1e0, 0xc01fc937a0, 0xc023653400, 0xc023653400, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01f486630, 0x4bc7440, 0xc002dfe1e0, 0x0, 0xc000100800, 0x0, 0xc006821620)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01f486630, 0x4bc7440, 0xc002dfe1e0, 0xc01fc937a0, 0xc0236533c0, 0xc011b69ac8, 0xca72c8, 0x48dd840, 0xc0236533c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01f486630, 0x4bc7440, 0xc002dfe1e0, 0xc01fc937a0, 0xc01fc937c0, 0xc007304ba0, 0x661046, 0x491f620, 0xc01f486630)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01f486630, 0xc0068215c0, 0xc001681820, 0x5a4e4e0, 0xc01f486630, 0xc00ff4cc00, 0x32d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0150e5800, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0150e5800, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0150e5800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218082215 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02c2fcc00, 0x1688c72a30e2872a, 0x1, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02c2fcc00, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02c2fcbd0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02c2fcbd0, 0x4c245c5, 0x10, 0xc02d7616c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02c2fcbd0, 0xc02c2e86e0, 0xc02c2fcbd0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02c2fcb40, 0xc02c2e86e0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02c2fcb40, 0x4bc7440, 0xc02c2e86e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02c2fcb40, 0x4bc7440, 0xc02c2e86e0, 0xc00dc227c0, 0xc00dc227e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02c2fcb40, 0x4bc7440, 0xc02c2e86e0, 0xc02d761990, 0xca72c8, 0x48dd840, 0xc007ea4180)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02c2fcb40, 0x4bc7440, 0xc02c2e86e0, 0xc00dc227c0, 0xc007ea4180, 0xc007ea4180, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02c2fcb40, 0x4bc7440, 0xc02c2e86e0, 0x0, 0xc000600000, 0x0, 0x7f8b0dd8f5c8)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02c2fcb40, 0x4bc7440, 0xc02c2e86e0, 0xc00dc227c0, 0xc007ea4140, 0xc02d761ac8, 0xca72c8, 0x48dd840, 0xc007ea4140)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02c2fcb40, 0x4bc7440, 0xc02c2e86e0, 0xc00dc227c0, 0xc00dc227e0, 0xc0013f1ba0, 0x661046, 0x491f620, 0xc02c2fcb40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02c2fcb40, 0xc02b9df2c0, 0xc001681820, 0x5a4e4e0, 0xc02c2fcb40, 0xc02d72ec00, 0x319)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc02c2fa400, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc02c2fa400, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc02c2fa400)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218080014 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02a8bd920, 0x1688c727ffe678ac, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02a8bd920, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02a8bd8f0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02a8bd8f0, 0x4c245c5, 0x10, 0xc01f55b6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02a8bd8f0, 0xc0236f1f40, 0xc02a8bd8f0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02a8bd860, 0xc0236f1f40, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02a8bd860, 0x4bc7440, 0xc0236f1f40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02a8bd860, 0x4bc7440, 0xc0236f1f40, 0xc01e353160, 0xc01e353180, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02a8bd860, 0x4bc7440, 0xc0236f1f40, 0xc01f55b990, 0xca72c8, 0x48dd840, 0xc01418b100)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02a8bd860, 0x4bc7440, 0xc0236f1f40, 0xc01e353160, 0xc01418b100, 0xc01418b100, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02a8bd860, 0x4bc7440, 0xc0236f1f40, 0x0, 0xc003d76000, 0x0, 0xc001c57800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02a8bd860, 0x4bc7440, 0xc0236f1f40, 0xc01e353160, 0xc01418b0c0, 0xc01f55bac8, 0xca72c8, 0x48dd840, 0xc01418b0c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02a8bd860, 0x4bc7440, 0xc0236f1f40, 0xc01e353160, 0xc01e353180, 0xc003a32ba0, 0x661046, 0x491f620, 0xc02a8bd860)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02a8bd860, 0xc001c577a0, 0xc001681820, 0x5a4e4e0, 0xc02a8bd860, 0xc01ebc5200, 0x300)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc029932000, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc029932000, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc029932000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217748462 [semacquire, 32 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc0015f2e78, 0x1688c57d2d68f31a, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc0015f2e78, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc016b2aea0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc016b2aea0, 0x4c245c5, 0x10, 0xc0278ae3f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc016b2aea0, 0xc01dc050e0, 0xc016b2aea0, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc016b2ae70, 0xc01dc050e0, 0xc0015f2b38, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc0122b5a00, 0x5a4e4e0, 0xc016b2ae70, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc0122b5a00, 0x5a6dfa0, 0xc0015f2b38, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc0015f2b38, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc0015f2b38, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc0015f2b38, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc0015f2b38, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc0015f2998, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc012e5e450, 0x5a6dfa0, 0xc0015f2998, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc012e5e418, 0x5a6dfa0, 0xc0015f2998, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc012e5e3e8, 0x5a6dfa0, 0xc0015f2998, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc012e5e348, 0x5a6dfa0, 0xc0015f2998, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc012e5e348, 0x5a6dfa0, 0xc0015f2998, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc012e5e278, 0x5a6dfa0, 0xc0015f2998, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc012e5e258, 0x5a6dfa0, 0xc0015f2998, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc012e5e1c0, 0x5a6dfa0, 0xc0015f2998, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc012e5e000, 0x5a4e420, 0xc01419a640, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc01419a640, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc014bbba70, 0x5a4e420, 0xc01419a640, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc01419a640, 0xc0278b31f0, 0xc00df68500, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc014bbba70, 0x5a4e420, 0xc01419a640, 0xc00df68500, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc014bbba70, 0x5a4e420, 0xc01419a640, 0xc00df68500, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc0011a3228, 0x5a4e420, 0xc01419a640, 0xc00f932c00, 0xc015d990e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc0011a3200, 0x5a4e420, 0xc01419a640, 0xc00f932c00, 0xc015d990e0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc016fd0ba0, 0x5a4e420, 0xc01419a640, 0xc00f932c00, 0xc015d990e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc016fd0ba0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc0278b3588, 0x4bfc8f0, 0x5, 0x5a50860, 0xc016fd0ba0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc0278b3588, 0x5a50860, 0xc016fd0ba0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc0278b3588, 0x5a50860, 0xc016fd0ba0, 0xc, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc01419a640, 0xc00f932c00, 0xc015d990e0, 0x5a50860, 0xc016fd0ba0, 0x5a4e420, 0xc01419a640)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc00d4a1200, 0x5a4e420, 0xc01419a640)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc00bae2400)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0278b3768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc0125d8c00, 0x5a4e420, 0xc01419a640)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc0125d8c00, 0x5a4e420, 0xc01419a640)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00f32c900, 0x5a4e420, 0xc01419a640, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc019244460, 0xc014bbba70, 0xc016ed1700, 0xc0061cbc00, 0xc015d992f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc01156fb78, 0xc015d992f8, 0xc019244460, 0xc014bbba70, 0x5a50860, 0xc016fd0ba0, 0x0, 0xc0061cbc00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc015d98c00, 0x5a6dfa0, 0xc01156fb78, 0xc015d990e0, 0x2, 0x7f8b0dd7dcb0, 0xc021739380, 0xc0278b4000, 0xc0230debf8, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc015d98c00, 0x5a6dfa0, 0xc01156fb78, 0xc015d990e0, 0x7f8b0dd7dcb0, 0xc021739380, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc015d98c00, 0x5a6dfa0, 0xc01156fb78, 0x5a6dda0, 0xc016ed0700, 0xc012363a0a, 0x1e9, 0x11, 0x1, 0xc0095fd3b0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc015d98c00, 0x5a4e420, 0xc015eb5240, 0x5a6dda0, 0xc016ed0700, 0xc012363a0a, 0x1e9, 0x11, 0x1, 0xc0095fd3b0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc015d98c00, 0x5a4e420, 0xc015eb5240, 0xc0095fd3b0, 0xc00f32c120, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc012363d70, 0x0, 0x0, 0x5641b3f, 0xed85aa647, 0x0, 0xc015d98c00, 0xc0278b5a20, 0xc0278b5a10, 0x35, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc015d98c00, 0x5a4e420, 0xc015eb5240, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc015d98c00, 0x5a4e420, 0xc0205b57c0, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc00637b5a0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc0205b57c0, 0xc015d98c00, 0x5400, 0x15000, 0xc001805a40, 0xc00637b5a0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc01ad3c456, 0xc00f241340, 0x5a4e420, 0xc0205b57c0, 0xc00637b5a0, 0xc001a38000, 0xc021738d00, 0x5ab5540, 0xc011b25080, 0xc00b201500, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216997600 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01a57db60, 0x1688c18c68591092, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01a57db60, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01a57db30, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01a57db30, 0x4c245c5, 0x10, 0xc0051296c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01a57db30, 0xc012483ae0, 0xc01a57db30, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01a57dad0, 0xc012483ae0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01a57dad0, 0x4bc7440, 0xc012483ae0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01a57dad0, 0x4bc7440, 0xc012483ae0, 0xc01fb8a820, 0xc01fb8a840, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01a57dad0, 0x4bc7440, 0xc012483ae0, 0xc005129990, 0xca72c8, 0x48dd840, 0xc00c4f9e00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01a57dad0, 0x4bc7440, 0xc012483ae0, 0xc01fb8a820, 0xc00c4f9e00, 0xc00c4f9e00, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01a57dad0, 0x4bc7440, 0xc012483ae0, 0x0, 0xc024c79800, 0x0, 0xc00ff1f680)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01a57dad0, 0x4bc7440, 0xc012483ae0, 0xc01fb8a820, 0xc00c4f9dc0, 0xc005129ac8, 0xca72c8, 0x48dd840, 0xc00c4f9dc0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01a57dad0, 0x4bc7440, 0xc012483ae0, 0xc01fb8a820, 0xc01fb8a840, 0xc02eb7bba0, 0x661046, 0x491f620, 0xc01a57dad0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01a57dad0, 0xc00ff1f4a0, 0xc001681820, 0x5a4e4e0, 0xc01a57dad0, 0xc016388600, 0x346)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc022b8fd00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc022b8fd00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc022b8fd00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216091229 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc014d995c0, 0x1688bcc2b7670983, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc014d995c0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc014d995c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc014d995c0, 0x4c245c5, 0x10, 0xc021764410, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc014d995c0, 0xc023d15ae0, 0xc014d995c0, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc014d99590, 0xc023d15ae0, 0xc0119267c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc014281a80, 0x5a4e4e0, 0xc014d99590, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc014281a80, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0127991d0, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc012799198, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc012799168, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc0127990c8, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc0127990c8, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc012798ff8, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc012798fd8, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc012798f40, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc012798d80, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc01f1af0e0, 0x5a4e420, 0xc0119267c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc0119267c0, 0xc021769208, 0xc00b317400, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc01f1af0e0, 0x5a4e420, 0xc0119267c0, 0xc00b317400, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc01f1af0e0, 0x5a4e420, 0xc0119267c0, 0xc00b317400, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc00473ac28, 0x5a4e420, 0xc0119267c0, 0xc00ed0c900, 0xc023963de0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00473ac00, 0x5a4e420, 0xc0119267c0, 0xc00ed0c900, 0xc023963de0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc0073246c0, 0x5a4e420, 0xc0119267c0, 0xc00ed0c900, 0xc023963de0, 0xc01e963400, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc0073246c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc0217695a0, 0x4bfc8f0, 0x5, 0x5a50860, 0xc0073246c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc01e9635a0, 0x5a50860, 0xc0073246c0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc01e9635a0, 0x5a50860, 0xc0073246c0, 0x0, 0xc0225caf00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc0119267c0, 0xc00ed0c900, 0xc023963de0, 0x5a50860, 0xc0073246c0, 0x5a4e420, 0xc0119267c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc021cb4c00, 0x5a4e420, 0xc0119267c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc002bc0900)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01e963768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc013243200, 0x5a4e420, 0xc0119267c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc013243200, 0x5a4e420, 0xc0119267c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc005f97c20, 0x5a4e420, 0xc0119267c0, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc0036d6b60, 0xc01f1af0e0, 0xc013323f80, 0xc015221500, 0xc023963ff8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc00f5241b0, 0xc023963ff8, 0xc0036d6b60, 0xc01f1af0e0, 0x5a50860, 0xc0073246c0, 0x0, 0xc015221500, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc023963900, 0x5a4e4e0, 0xc00f5241b0, 0xc023963de0, 0x2, 0x7f8b0dd7dcb0, 0xc021736680, 0xc01e964000, 0xc015578d78, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc023963900, 0x5a4e4e0, 0xc00f5241b0, 0xc023963de0, 0x7f8b0dd7dcb0, 0xc021736680, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc023963900, 0x5a4e4e0, 0xc00f5241b0, 0x5a6dda0, 0xc013323a80, 0xc009b6e035, 0x1e9, 0x11, 0x1, 0xc012b32d20, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc023963900, 0x5a4e420, 0xc01a82e580, 0x5a6dda0, 0xc013323a80, 0xc009b6e035, 0x1e9, 0x11, 0x1, 0xc012b32d20, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc023963900, 0x5a4e420, 0xc01a82e580, 0xc012b32d20, 0xc005f97320, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc009b6e3b3, 0x0, 0x0, 0x928dfe1, 0xed85a80ca, 0x0, 0xc023963900, 0xc02176ba20, 0xc02176ba10, 0x5, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc023963900, 0x5a4e420, 0xc01a82e580, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc023963900, 0x5a4e420, 0xc00ff96e80, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc009534630, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc00ff96e80, 0xc023963900, 0x5400, 0x15000, 0xc001805a40, 0xc009534630, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc00f43e3a1, 0xc01e775840, 0x5a4e420, 0xc00ff96e80, 0xc009534630, 0xc001a38000, 0xc021736000, 0x5ab5540, 0xc023218580, 0xc01594e2a0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216370368 [semacquire, 169 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc008aa5aa0, 0x1688be03fdc5c204, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc008aa5aa0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc008aa5a70, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc008aa5a70, 0x4c245c5, 0x10, 0xc0238936c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc008aa5a70, 0xc02190ed20, 0xc008aa5a70, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc008aa5a10, 0xc02190ed20, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc008aa5a10, 0x4bc7440, 0xc02190ed20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc008aa5a10, 0x4bc7440, 0xc02190ed20, 0xc0101ab8e0, 0xc0101ab900, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc008aa5a10, 0x4bc7440, 0xc02190ed20, 0xc023893990, 0xca72c8, 0x48dd840, 0xc010395e00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc008aa5a10, 0x4bc7440, 0xc02190ed20, 0xc0101ab8e0, 0xc010395e00, 0xc010395e00, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc008aa5a10, 0x4bc7440, 0xc02190ed20, 0x0, 0xc003d76000, 0x0, 0xc024d057a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc008aa5a10, 0x4bc7440, 0xc02190ed20, 0xc0101ab8e0, 0xc010395dc0, 0xc023893ac8, 0xca72c8, 0x48dd840, 0xc010395dc0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc008aa5a10, 0x4bc7440, 0xc02190ed20, 0xc0101ab8e0, 0xc0101ab900, 0xc0037e3ba0, 0x661046, 0x491f620, 0xc008aa5a10)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc008aa5a10, 0xc024d05740, 0xc001681820, 0x5a4e4e0, 0xc008aa5a10, 0xc00cc1f200, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0096e5000, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0096e5000, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0096e5000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218079901 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02986b4a0, 0x1688c727fb96c4ba, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02986b4a0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02986b470, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02986b470, 0x4c245c5, 0x10, 0xc01efa16c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02986b470, 0xc02a0af9a0, 0xc02986b470, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02986b410, 0xc02a0af9a0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02986b410, 0x4bc7440, 0xc02a0af9a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02986b410, 0x4bc7440, 0xc02a0af9a0, 0xc01e52cba0, 0xc01e52cbc0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02986b410, 0x4bc7440, 0xc02a0af9a0, 0xc01efa1990, 0xca72c8, 0x48dd840, 0xc01ea36cc0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02986b410, 0x4bc7440, 0xc02a0af9a0, 0xc01e52cba0, 0xc01ea36cc0, 0xc01ea36cc0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02986b410, 0x4bc7440, 0xc02a0af9a0, 0x0, 0xc003d76000, 0x0, 0xc007aa1ce0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02986b410, 0x4bc7440, 0xc02a0af9a0, 0xc01e52cba0, 0xc01ea36c80, 0xc01efa1ac8, 0xca72c8, 0x48dd840, 0xc01ea36c80)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02986b410, 0x4bc7440, 0xc02a0af9a0, 0xc01e52cba0, 0xc01e52cbc0, 0xc003019ba0, 0x661046, 0x491f620, 0xc02986b410)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02986b410, 0xc007aa1c80, 0xc001681820, 0x5a4e4e0, 0xc02986b410, 0xc01e92c000, 0x33a8)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01abf1000, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01abf1000, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01abf1000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218082008 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc029f75cb0, 0x1688c72a29c834e5, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc029f75cb0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc029f75c80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc029f75c80, 0x4c245c5, 0x10, 0xc02a78d6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc029f75c80, 0xc02a14a500, 0xc029f75c80, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc029f75bf0, 0xc02a14a500, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc029f75bf0, 0x4bc7440, 0xc02a14a500, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc029f75bf0, 0x4bc7440, 0xc02a14a500, 0xc0210e5b20, 0xc0210e5b40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc029f75bf0, 0x4bc7440, 0xc02a14a500, 0xc02a78d990, 0xca72c8, 0x48dd840, 0xc01a8b6880)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc029f75bf0, 0x4bc7440, 0xc02a14a500, 0xc0210e5b20, 0xc01a8b6880, 0xc01a8b6880, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc029f75bf0, 0x4bc7440, 0xc02a14a500, 0x0, 0xc00e8e4800, 0x0, 0xc02a1662a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc029f75bf0, 0x4bc7440, 0xc02a14a500, 0xc0210e5b20, 0xc01a8b6840, 0xc02a78dac8, 0xca72c8, 0x48dd840, 0xc01a8b6840)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc029f75bf0, 0x4bc7440, 0xc02a14a500, 0xc0210e5b20, 0xc0210e5b40, 0xc0020f4ba0, 0x661046, 0x491f620, 0xc029f75bf0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc029f75bf0, 0xc02a166240, 0xc001681820, 0x5a4e4e0, 0xc029f75bf0, 0xc02a13d800, 0x2db)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02a13a300, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02a13a300, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02a13a300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217230997 [semacquire, 84 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc00863c660, 0x1688c2ae714eea46, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc00863c660, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00863c630, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00863c630, 0x4c245c5, 0x10, 0xc0041a96c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00863c630, 0xc0225d6aa0, 0xc00863c630, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00863c240, 0xc0225d6aa0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc00863c240, 0x4bc7440, 0xc0225d6aa0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc00863c240, 0x4bc7440, 0xc0225d6aa0, 0xc01977b4c0, 0xc01977b4e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00863c240, 0x4bc7440, 0xc0225d6aa0, 0xc0041a9990, 0xca72c8, 0x48dd840, 0xc01284c8c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc00863c240, 0x4bc7440, 0xc0225d6aa0, 0xc01977b4c0, 0xc01284c8c0, 0xc01284c8c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00863c240, 0x4bc7440, 0xc0225d6aa0, 0x0, 0xc000600000, 0x0, 0xc00c295aa0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc00863c240, 0x4bc7440, 0xc0225d6aa0, 0xc01977b4c0, 0xc01284c880, 0xc0041a9ac8, 0xca72c8, 0x48dd840, 0xc01284c880)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc00863c240, 0x4bc7440, 0xc0225d6aa0, 0xc01977b4c0, 0xc01977b4e0, 0xc01e93eba0, 0x661046, 0x491f620, 0xc00863c240)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc00863c240, 0xc00c2959e0, 0xc001681820, 0x5a4e4e0, 0xc00863c240, 0xc0088b8000, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02097c200, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02097c200, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02097c200)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216449916 [semacquire, 162 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0223391a0, 0x1688be65c82ce50f, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0223391a0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc022339140, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc022339140, 0x4c245c5, 0x10, 0xc004dab6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc022339140, 0xc011bc57c0, 0xc022339140, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0223390e0, 0xc011bc57c0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0223390e0, 0x4bc7440, 0xc011bc57c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0223390e0, 0x4bc7440, 0xc011bc57c0, 0xc001d85880, 0xc001d858e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0223390e0, 0x4bc7440, 0xc011bc57c0, 0xc004dab990, 0xca72c8, 0x48dd840, 0xc010435580)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0223390e0, 0x4bc7440, 0xc011bc57c0, 0xc001d85880, 0xc010435580, 0xc010435580, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0223390e0, 0x4bc7440, 0xc011bc57c0, 0x0, 0xc003d76000, 0x0, 0xc007e299e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0223390e0, 0x4bc7440, 0xc011bc57c0, 0xc001d85880, 0xc010435500, 0xc004dabac8, 0xca72c8, 0x48dd840, 0xc010435500)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0223390e0, 0x4bc7440, 0xc011bc57c0, 0xc001d85880, 0xc001d858e0, 0xc0000acba0, 0x661046, 0x491f620, 0xc0223390e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0223390e0, 0xc007e29980, 0xc001681820, 0x5a4e4e0, 0xc0223390e0, 0xc016d44c00, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc004d8a000, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc004d8a000, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc004d8a000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217885490 [semacquire, 21 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01d581c50, 0x1688c61504ca4978, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01d581c50, 0x1688c61504ca4978, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01d581c20, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01d581c20, 0x4c245c5, 0x10, 0xc009fc76c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01d581c20, 0xc00812cd20, 0xc01d581c20, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01d581230, 0xc00812cd20, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01d581230, 0x4bc7440, 0xc00812cd20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01d581230, 0x4bc7440, 0xc00812cd20, 0xc0085c9aa0, 0xc0085c9ac0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01d581230, 0x4bc7440, 0xc00812cd20, 0xc009fc7990, 0xca72c8, 0x48dd840, 0xc006551640)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01d581230, 0x4bc7440, 0xc00812cd20, 0xc0085c9aa0, 0xc006551640, 0xc006551640, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01d581230, 0x4bc7440, 0xc00812cd20, 0x0, 0xc00e8e4800, 0x0, 0xc011ae3aa0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01d581230, 0x4bc7440, 0xc00812cd20, 0xc0085c9aa0, 0xc006551480, 0xc009fc7ac8, 0xca72c8, 0x48dd840, 0xc006551480)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01d581230, 0x4bc7440, 0xc00812cd20, 0xc0085c9aa0, 0xc0085c9ac0, 0xc02f07bba0, 0x661046, 0x491f620, 0xc01d581230)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01d581230, 0xc011ae3a40, 0xc001681820, 0x5a4e4e0, 0xc01d581230, 0xc00c45a000, 0x419)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f81b500, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f81b500, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f81b500)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217675819 [semacquire, 42 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc012b0e800, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc012b0e870, 0xc00ab8f388)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:372 +0x1db
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).ComputeStatsForKeySpan(0xc0018ad000, 0xc011a3b0f0, 0x2, 0x8, 0xc011a3b0f8, 0x4, 0x8, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2710 +0x15f
github.com/cockroachdb/cockroach/pkg/server.(*statusServer).SpanStats.func1(0xc0018ad000, 0x464925, 0xc01e93e658)
/go/src/github.com/cockroachdb/cockroach/pkg/server/status.go:2273 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores.func1(0x1, 0xc0018ad000, 0xc01e93e658)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:148 +0x38
github.com/cockroachdb/cockroach/pkg/util/syncutil.(*IntMap).Range(0xc0002f7c30, 0xc00ab8f6e8)
/go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/int_map.go:352 +0x130
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores(0xc0002f7c00, 0xc01e93e768, 0x1, 0x100000001)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:147 +0x75
github.com/cockroachdb/cockroach/pkg/server.(*statusServer).SpanStats(0xc00589e840, 0x5a4e4e0, 0xc012b0e810, 0xc021180d00, 0xc00589e840, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/status.go:2272 +0x170
github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_SpanStats_Handler.func1(0x5a4e4e0, 0xc012b0e720, 0x4a19420, 0xc021180d00, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:4761 +0x8b
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc012b0e720, 0x4a19420, 0xc021180d00, 0xc00a10f7e0, 0xc00a10f800, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc012b0e720, 0x4a19420, 0xc021180d00, 0xc01e93e990, 0xca72c8, 0x48dd840, 0xc021181040)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc012b0e720, 0x4a19420, 0xc021180d00, 0xc00a10f7e0, 0xc021181040, 0xc021181040, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc012b0e720, 0x4a19420, 0xc021180d00, 0x0, 0xc00009c800, 0x0, 0x7f8b0ddea548)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc012b0e720, 0x4a19420, 0xc021180d00, 0xc00a10f7e0, 0xc021180d40, 0xc01e93eac8, 0xca72c8, 0x48dd840, 0xc021180d40)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc012b0e720, 0x4a19420, 0xc021180d00, 0xc00a10f7e0, 0xc00a10f800, 0xc01e93eba0, 0x661046, 0x491f620, 0xc012b0e720)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_SpanStats_Handler(0x4bcfa40, 0xc00589e840, 0x5a4e4e0, 0xc012b0e720, 0xc01a444fc0, 0xc001681820, 0x5a4e4e0, 0xc012b0e720, 0xc01d5f9800, 0xc)
/go/src/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:4763 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00b50e500, 0xc00076a8a0, 0x7cd3c80, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00b50e500, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00b50e500)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 4880 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0022b7e00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x28a
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10fb
goroutine 4873 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0037876d0, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x125
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc003780b40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1d3
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0022b7c80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xd7
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10d6
goroutine 5138 [IO wait]:
internal/poll.runtime_pollWait(0x7f8b125caf48, 0x72, 0x59b54a0)
/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc003166798, 0x72, 0x59b5400, 0x7cf1b20, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc003166780, 0xc00a74ca00, 0x510c, 0x510c, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:159 +0x1a5
net.(*netFD).Read(0xc003166780, 0xc00a74ca00, 0x510c, 0x510c, 0xc, 0xc, 0xc00a74ca05)
/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc0032e7a80, 0xc00a74ca00, 0x510c, 0x510c, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:182 +0x8e
github.com/cockroachdb/cmux.(*MuxConn).Read(0xc004a7b920, 0xc00a74ca00, 0x510c, 0x510c, 0xc00a74ca00, 0x5, 0x7f8ab12bb778)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x115
crypto/tls.(*atLeastReader).Read(0xc01b217f80, 0xc00a74ca00, 0x510c, 0x510c, 0x2b, 0x5107, 0xc007307798)
/usr/local/go/src/crypto/tls/conn.go:779 +0x62
bytes.(*Buffer).ReadFrom(0xc001602980, 0x59a7c60, 0xc01b217f80, 0x45f8c5, 0x458c0a0, 0x49151a0)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc001602700, 0x7f8b12553238, 0xc004a7b920, 0x5, 0xc004a7b920, 0x1a)
/usr/local/go/src/crypto/tls/conn.go:801 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc001602700, 0x0, 0x0, 0xc3213e)
/usr/local/go/src/crypto/tls/conn.go:608 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:576
crypto/tls.(*Conn).Read(0xc001602700, 0xc0026f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1252 +0x15f
bufio.(*Reader).Read(0xc0035c02a0, 0xc0011cec78, 0x9, 0x9, 0x46ffb00, 0x4, 0xc000100400)
/usr/local/go/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x59a7a20, 0xc0035c02a0, 0xc0011cec78, 0x9, 0x9, 0x9, 0x18, 0xc007307dd0, 0x462518)
/usr/local/go/src/io/io.go:314 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:333
golang.org/x/net/http2.readFrameHeader(0xc0011cec78, 0x9, 0x9, 0x59a7a20, 0xc0035c02a0, 0x0, 0xc000000000, 0xc002717e90, 0x44e0040)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011cec40, 0xc01a9d44b0, 0x8702740, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc003d78300, 0xc002a68a80, 0x52fddb8)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0005601a0, 0x5aba220, 0xc003d78300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005601a0, 0x5aba220, 0xc003d78300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:702 +0x50b
goroutine 134774 [semacquire]:
sync.runtime_Semacquire(0xc010a6fbe8)
/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc010a6fbe0)
/usr/local/go/src/sync/waitgroup.go:130 +0x65
google.golang.org/grpc.(*Server).serveStreams(0xc0005601a0, 0x5aba220, 0xc009a50180)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:755 +0xe9
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005601a0, 0x5aba220, 0xc009a50180)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:702 +0x50b
goroutine 216996395 [chan receive, 32 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc012f36000, 0x5a4e4e0, 0xc020a903f0, 0xc0072f7520, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc020a903f0, 0x5ab2120, 0xc00f2fbc00, 0xc00b44f028, 0xd, 0xc020a903c0, 0x59fb820, 0xc01544d2c0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc020a903f0, 0x5ab21e0, 0xc002fd38c0, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc002fd38c0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc002fd38c0, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 217769514 [semacquire, 30 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc023896f60, 0x1688c598ad9fbe8b, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc023896f60, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc023896f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc023896f00, 0x4c245c5, 0x10, 0xc0051bf6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc023896f00, 0xc020591220, 0xc023896f00, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc023896c90, 0xc020591220, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc023896c90, 0x4bc7440, 0xc020591220, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc023896c90, 0x4bc7440, 0xc020591220, 0xc01a5143c0, 0xc01a5143e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc023896c90, 0x4bc7440, 0xc020591220, 0xc0051bf990, 0xca72c8, 0x48dd840, 0xc021534d80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc023896c90, 0x4bc7440, 0xc020591220, 0xc01a5143c0, 0xc021534d80, 0xc021534d80, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc023896c90, 0x4bc7440, 0xc020591220, 0x0, 0xc0002a9800, 0x0, 0xc016836b40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc023896c90, 0x4bc7440, 0xc020591220, 0xc01a5143c0, 0xc021534d40, 0xc0051bfac8, 0xca72c8, 0x48dd840, 0xc021534d40)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc023896c90, 0x4bc7440, 0xc020591220, 0xc01a5143c0, 0xc01a5143e0, 0xc00187dba0, 0x661046, 0x491f620, 0xc023896c90)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc023896c90, 0xc016836a80, 0xc001681820, 0x5a4e4e0, 0xc023896c90, 0xc005112600, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0107f4300, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0107f4300, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0107f4300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216997585 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01e8bd170, 0x1688c18c6c73bd2b, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01e8bd170, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01e8bd140, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01e8bd140, 0x4c245c5, 0x10, 0xc0059a76c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01e8bd140, 0xc0068170e0, 0xc01e8bd140, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01e8bd020, 0xc0068170e0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01e8bd020, 0x4bc7440, 0xc0068170e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01e8bd020, 0x4bc7440, 0xc0068170e0, 0xc00bce01c0, 0xc00bce01e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01e8bd020, 0x4bc7440, 0xc0068170e0, 0xc0059a7990, 0xca72c8, 0x48dd840, 0xc019ab1000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01e8bd020, 0x4bc7440, 0xc0068170e0, 0xc00bce01c0, 0xc019ab1000, 0xc019ab1000, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01e8bd020, 0x4bc7440, 0xc0068170e0, 0x0, 0xc00e8e4800, 0x0, 0xc00801ecc0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01e8bd020, 0x4bc7440, 0xc0068170e0, 0xc00bce01c0, 0xc019ab0fc0, 0xc0059a7ac8, 0xca72c8, 0x48dd840, 0xc019ab0fc0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01e8bd020, 0x4bc7440, 0xc0068170e0, 0xc00bce01c0, 0xc00bce01e0, 0xc003c28ba0, 0x661046, 0x491f620, 0xc01e8bd020)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01e8bd020, 0xc00801ec60, 0xc001681820, 0x5a4e4e0, 0xc01e8bd020, 0xc00af52000, 0x30e)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00e711b00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00e711b00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00e711b00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216997675 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc001e69410, 0x1688c18c6747ac98, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc001e69410, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc001e69380, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc001e69380, 0x4c245c5, 0x10, 0xc0055df6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc001e69380, 0xc012482a00, 0xc001e69380, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc001e68e40, 0xc012482a00, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc001e68e40, 0x4bc7440, 0xc012482a00, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc001e68e40, 0x4bc7440, 0xc012482a00, 0xc01f63efa0, 0xc01f63efc0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc001e68e40, 0x4bc7440, 0xc012482a00, 0xc0055df990, 0xca72c8, 0x48dd840, 0xc010f1aa80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc001e68e40, 0x4bc7440, 0xc012482a00, 0xc01f63efa0, 0xc010f1aa80, 0xc010f1aa80, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc001e68e40, 0x4bc7440, 0xc012482a00, 0x0, 0xc024c79800, 0x0, 0xc00708fbc0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc001e68e40, 0x4bc7440, 0xc012482a00, 0xc01f63efa0, 0xc010f1aa40, 0xc0055dfac8, 0xca72c8, 0x48dd840, 0xc010f1aa40)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc001e68e40, 0x4bc7440, 0xc012482a00, 0xc01f63efa0, 0xc01f63efc0, 0xc02eb7bba0, 0x661046, 0x491f620, 0xc001e68e40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc001e68e40, 0xc00708fb60, 0xc001681820, 0x5a4e4e0, 0xc001e68e40, 0xc01ed31800, 0x36d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc014ed6700, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc014ed6700, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc014ed6700)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216321765 [semacquire, 173 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc015a60990, 0x1688bdc89e4a31d5, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc015a60990, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc015a60960, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc015a60960, 0x4c245c5, 0x10, 0xc006d8d6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc015a60960, 0xc002a0cf00, 0xc015a60960, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc015a60900, 0xc002a0cf00, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc015a60900, 0x4bc7440, 0xc002a0cf00, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc015a60900, 0x4bc7440, 0xc002a0cf00, 0xc0034f8300, 0xc0034f8320, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc015a60900, 0x4bc7440, 0xc002a0cf00, 0xc006d8d990, 0xca72c8, 0x48dd840, 0xc014b6c940)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc015a60900, 0x4bc7440, 0xc002a0cf00, 0xc0034f8300, 0xc014b6c940, 0xc014b6c940, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc015a60900, 0x4bc7440, 0xc002a0cf00, 0x0, 0x87067e0, 0x0, 0xc0150a77a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc015a60900, 0x4bc7440, 0xc002a0cf00, 0xc0034f8300, 0xc014b6c8c0, 0xc006d8dac8, 0xca72c8, 0x48dd840, 0xc014b6c8c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc015a60900, 0x4bc7440, 0xc002a0cf00, 0xc0034f8300, 0xc0034f8320, 0xc00190aba0, 0x661046, 0x491f620, 0xc015a60900)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc015a60900, 0xc0150a7740, 0xc001681820, 0x5a4e4e0, 0xc015a60900, 0xc00440c000, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0062df100, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0062df100, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0062df100)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216996925 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01326d1d0, 0x1688c18bc28164b9, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01326d1d0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01326d1a0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01326d1a0, 0x4c245c5, 0x10, 0xc0043a16c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01326d1a0, 0xc014c6a3c0, 0xc01326d1a0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01326d140, 0xc014c6a3c0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01326d140, 0x4bc7440, 0xc014c6a3c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01326d140, 0x4bc7440, 0xc014c6a3c0, 0xc009866040, 0xc009866060, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01326d140, 0x4bc7440, 0xc014c6a3c0, 0xc0043a1990, 0xca72c8, 0x48dd840, 0xc01dbda780)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01326d140, 0x4bc7440, 0xc014c6a3c0, 0xc009866040, 0xc01dbda780, 0xc01dbda780, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01326d140, 0x4bc7440, 0xc014c6a3c0, 0x0, 0xc024c79800, 0x0, 0xc00f810de0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01326d140, 0x4bc7440, 0xc014c6a3c0, 0xc009866040, 0xc01dbda740, 0xc0043a1ac8, 0xca72c8, 0x48dd840, 0xc01dbda740)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01326d140, 0x4bc7440, 0xc014c6a3c0, 0xc009866040, 0xc009866060, 0xc007306ba0, 0x661046, 0x491f620, 0xc01326d140)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01326d140, 0xc00f810d80, 0xc001681820, 0x5a4e4e0, 0xc01326d140, 0xc014c62c00, 0x32a)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00e72c100, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00e72c100, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00e72c100)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 6579 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc02152e800, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc02152e8a0, 0xc0157a4f88)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:372 +0x1db
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Capacity(0xc0018ad000, 0x5a4e4e0, 0xc002fbe360, 0x6c19c700, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2385 +0x4ed
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Descriptor(0xc0018ad000, 0x5a4e4e0, 0xc002fbe360, 0xc000c43500, 0x0, 0x2b, 0x30)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2455 +0x69
github.com/cockroachdb/cockroach/pkg/server/status.(*MetricsRecorder).GenerateNodeStatus(0xc000602f00, 0x5a4e4e0, 0xc002fbe360, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/status/recorder.go:467 +0x7a9
github.com/cockroachdb/cockroach/pkg/server.(*Node).writeNodeStatus.func1(0x5a4e4e0, 0xc002fbe360)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:753 +0x8b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000c04b80, 0x5a4e4e0, 0xc002fbe360, 0x4c523dc, 0x1a, 0xc0157a5e20, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:313 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).writeNodeStatus(0xc0058cac00, 0x5a4e4e0, 0xc002fbe360, 0x4a817c800, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:752 +0xbd
github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func1(0x5a4e4e0, 0xc002fbe360)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:736 +0x16d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc002fbe360, 0x0, 0xc008195980)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 218082011 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02a4ae360, 0x1688c72a2acc6363, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02a4ae360, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02a4ae330, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02a4ae330, 0x4c245c5, 0x10, 0xc02aa336c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02a4ae330, 0xc02a14b360, 0xc02a4ae330, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02a4ae2a0, 0xc02a14b360, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02a4ae2a0, 0x4bc7440, 0xc02a14b360, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02a4ae2a0, 0x4bc7440, 0xc02a14b360, 0xc014b07380, 0xc014b073a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02a4ae2a0, 0x4bc7440, 0xc02a14b360, 0xc02aa33990, 0xca72c8, 0x48dd840, 0xc0130bc300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02a4ae2a0, 0x4bc7440, 0xc02a14b360, 0xc014b07380, 0xc0130bc300, 0xc0130bc300, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02a4ae2a0, 0x4bc7440, 0xc02a14b360, 0x0, 0xc00e8e4800, 0x0, 0xc02a1675c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02a4ae2a0, 0x4bc7440, 0xc02a14b360, 0xc014b07380, 0xc0130bc2c0, 0xc02aa33ac8, 0xca72c8, 0x48dd840, 0xc0130bc2c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02a4ae2a0, 0x4bc7440, 0xc02a14b360, 0xc014b07380, 0xc014b073a0, 0xc0020f4ba0, 0x661046, 0x491f620, 0xc02a4ae2a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02a4ae2a0, 0xc02a167560, 0xc001681820, 0x5a4e4e0, 0xc02a4ae2a0, 0xc02a49d200, 0x310)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02a13aa00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02a13aa00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02a13aa00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216087059 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0235e94d0, 0x1688bcc1163a4644, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0235e94d0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0235e94d0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0235e94d0, 0x4c245c5, 0x10, 0xc015dd2410, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0235e94d0, 0xc014acc5a0, 0xc0235e94d0, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0235e94a0, 0xc014acc5a0, 0xc012e64300, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc012808ac0, 0x5a4e4e0, 0xc0235e94a0, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc012808ac0, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc008bf1f50, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc008bf1f18, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc008bf1ee8, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc008bf1e48, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc008bf1e48, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc008bf1d78, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc008bf1d58, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc008bf1cc0, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc008bf1b00, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00fb80000, 0x5a4e420, 0xc012e64300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc012e64300, 0xc015dd7208, 0xc00cafb900, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc00fb80000, 0x5a4e420, 0xc012e64300, 0xc00cafb900, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc00fb80000, 0x5a4e420, 0xc012e64300, 0xc00cafb900, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc015062c28, 0x5a4e420, 0xc012e64300, 0xc00e01ec00, 0xc01aa217e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc015062c00, 0x5a4e420, 0xc012e64300, 0xc00e01ec00, 0xc01aa217e0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc016e6e1e0, 0x5a4e420, 0xc012e64300, 0xc00e01ec00, 0xc01aa217e0, 0xc015dd7400, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc016e6e1e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc015dd75a0, 0x4bfc8f0, 0x5, 0x5a50860, 0xc016e6e1e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc015dd75a0, 0x5a50860, 0xc016e6e1e0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc015dd75a0, 0x5a50860, 0xc016e6e1e0, 0x0, 0xc00dbf39a0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc012e64300, 0xc00e01ec00, 0xc01aa217e0, 0x5a50860, 0xc016e6e1e0, 0x5a4e420, 0xc012e64300)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc00c728000, 0x5a4e420, 0xc012e64300)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc012749200)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc015dd7768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc00c729800, 0x5a4e420, 0xc012e64300)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc00c729800, 0x5a4e420, 0xc012e64300)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc0027d79e0, 0x5a4e420, 0xc012e64300, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc008d29960, 0xc00fb80000, 0xc019dcaa00, 0xc00b7be700, 0xc01aa219f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc007051080, 0xc01aa219f8, 0xc008d29960, 0xc00fb80000, 0x5a50860, 0xc016e6e1e0, 0x0, 0xc00b7be700, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc01aa21300, 0x5a4e4e0, 0xc007051080, 0xc01aa217e0, 0x2, 0x7f8b0dd7dcb0, 0xc0123b4f80, 0xc015dd8000, 0xc01e2c4658, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc01aa21300, 0x5a4e4e0, 0xc007051080, 0xc01aa217e0, 0x7f8b0dd7dcb0, 0xc0123b4f80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc01aa21300, 0x5a4e4e0, 0xc007051080, 0x5a6dda0, 0xc010d16e80, 0xc01292d015, 0x1e9, 0x11, 0x1, 0xc013102e10, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc01aa21300, 0x5a4e420, 0xc019d714c0, 0x5a6dda0, 0xc010d16e80, 0xc01292d015, 0x1e9, 0x11, 0x1, 0xc013102e10, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc01aa21300, 0x5a4e420, 0xc019d714c0, 0xc013102e10, 0xc0159d7440, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc01292d389, 0x0, 0x0, 0x93753a3, 0xed85a80c3, 0x0, 0xc01aa21300, 0xc015dd9a20, 0xc015dd9a10, 0x71, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc01aa21300, 0x5a4e420, 0xc019d714c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc01aa21300, 0x5a4e420, 0xc00cf26700, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc0133bbf00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc00cf26700, 0xc01aa21300, 0x5400, 0x15000, 0xc001805a40, 0xc0133bbf00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc0028af416, 0xc01f207d20, 0x5a4e420, 0xc00cf26700, 0xc0133bbf00, 0xc001a38000, 0xc0123b4900, 0x5ab5540, 0xc004c99c00, 0xc00c2666c0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216086338 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc00ad49740, 0x1688bcc09bed8600, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc00ad49740, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00ad49740, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00ad49740, 0x4c245c5, 0x10, 0xc003448410, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00ad49740, 0xc008074000, 0xc00ad49740, 0xa0, 0x88)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00ad48ba0, 0xc008074000, 0xc014c7f540, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00dd25740, 0x5a4e4e0, 0xc00ad48ba0, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00dd25740, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc014c0d650, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc014c0d618, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc014c0d5e8, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc014c0d548, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc014c0d548, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc014c0d478, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc014c0d458, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc014c0d3c0, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc014c0d200, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00e55db00, 0x5a4e420, 0xc014c7f540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc014c7f540, 0xc00344d208, 0xc0152c4000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc00e55db00, 0x5a4e420, 0xc014c7f540, 0xc0152c4000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc00e55db00, 0x5a4e420, 0xc014c7f540, 0xc0152c4000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc01f566628, 0x5a4e420, 0xc014c7f540, 0xc009626300, 0xc00d221de0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc01f566600, 0x5a4e420, 0xc014c7f540, 0xc009626300, 0xc00d221de0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc01aa0e840, 0x5a4e420, 0xc014c7f540, 0xc009626300, 0xc00d221de0, 0x4c4c00, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc01aa0e840, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc00344d5a0, 0x4bfc8f0, 0x5, 0x5a50860, 0xc01aa0e840)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc0052035a0, 0x5a50860, 0xc01aa0e840, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc0052035a0, 0x5a50860, 0xc01aa0e840, 0x0, 0xc005203620)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc014c7f540, 0xc009626300, 0xc00d221de0, 0x5a50860, 0xc01aa0e840, 0x5a4e420, 0xc014c7f540)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc014c06c00, 0x5a4e420, 0xc014c7f540)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc015027b00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc005203768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc014c07800, 0x5a4e420, 0xc014c7f540)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc014c07800, 0x5a4e420, 0xc014c7f540)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc014ebe480, 0x5a4e420, 0xc014c7f540, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc0133217a0, 0xc00e55db00, 0xc012723f80, 0xc0155ba000, 0xc00d221ff8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc0167b7620, 0xc00d221ff8, 0xc0133217a0, 0xc00e55db00, 0x5a50860, 0xc01aa0e840, 0x0, 0xc0155ba000, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00d221900, 0x5a4e4e0, 0xc0167b7620, 0xc00d221de0, 0x2, 0x7f8b0dd7dcb0, 0xc014d9b880, 0xc005204000, 0xc01e1f13d8, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00d221900, 0x5a4e4e0, 0xc0167b7620, 0xc00d221de0, 0x7f8b0dd7dcb0, 0xc014d9b880, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00d221900, 0x5a4e4e0, 0xc0167b7620, 0x5a6dda0, 0xc0109bb100, 0xc01354d944, 0x1e9, 0x11, 0x1, 0xc029b552c0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00d221900, 0x5a4e420, 0xc00d5d2f00, 0x5a6dda0, 0xc0109bb100, 0xc01354d944, 0x1e9, 0x11, 0x1, 0xc029b552c0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00d221900, 0x5a4e420, 0xc00d5d2f00, 0xc029b552c0, 0xc013d79c20, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc01354dcb3, 0x0, 0x0, 0x6200a36, 0xed85a80c1, 0x0, 0xc00d221900, 0xc00344fa20, 0xc00344fa10, 0x6e, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00d221900, 0x5a4e420, 0xc00d5d2f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00d221900, 0x5a4e420, 0xc015a4d280, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc00ffe01c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc015a4d280, 0xc00d221900, 0x5400, 0x15000, 0xc001805a40, 0xc00ffe01c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc011785441, 0xc0227557e0, 0x5a4e420, 0xc015a4d280, 0xc00ffe01c0, 0xc001a38000, 0xc014d9b200, 0x5ab5540, 0xc019bb4f80, 0xc00b497800, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 217316421 [semacquire, 76 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01e192f60, 0x1688c32088119131, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01e192f60, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01e192f30, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01e192f30, 0x4c245c5, 0x10, 0xc0072336c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01e192f30, 0xc00f716140, 0xc01e192f30, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01e192ed0, 0xc00f716140, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01e192ed0, 0x4bc7440, 0xc00f716140, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01e192ed0, 0x4bc7440, 0xc00f716140, 0xc008846a40, 0xc008846a60, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01e192ed0, 0x4bc7440, 0xc00f716140, 0xc007233990, 0xca72c8, 0x48dd840, 0xc02219c800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01e192ed0, 0x4bc7440, 0xc00f716140, 0xc008846a40, 0xc02219c800, 0xc02219c800, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01e192ed0, 0x4bc7440, 0xc00f716140, 0x0, 0xc00758a000, 0x0, 0xc020349c80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01e192ed0, 0x4bc7440, 0xc00f716140, 0xc008846a40, 0xc02219c7c0, 0xc007233ac8, 0xca72c8, 0x48dd840, 0xc02219c7c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01e192ed0, 0x4bc7440, 0xc00f716140, 0xc008846a40, 0xc008846a60, 0xc003a34ba0, 0x661046, 0x491f620, 0xc01e192ed0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01e192ed0, 0xc020349c20, 0xc001681820, 0x5a4e4e0, 0xc01e192ed0, 0xc002357800, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00a31b500, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00a31b500, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00a31b500)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 4879 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000112cd0, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x125
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00371f140, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1d3
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0022b7e00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xd7
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10d6
goroutine 217757111 [semacquire, 31 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0029f0840, 0x1688c5897fca11de, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0029f0840, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0029f0840, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0029f0840, 0x4c245c5, 0x10, 0xc006c8f6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0029f0840, 0xc01d554000, 0xc0029f0840, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a6dfa0, 0xc005ab6318, 0xc01d554000, 0xc0058cac00, 0xc0029f0540, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a6dfa0, 0xc005ab6318, 0x4bc7440, 0xc01d554000, 0x21, 0xc006c8f890, 0x2, 0x2)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0029f0270, 0x4bc7440, 0xc01d554000, 0xc01ace4cc0, 0xc01ace4d40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:137 +0x2dc
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0029f0270, 0x4bc7440, 0xc01d554000, 0xc006c8f990, 0xca72c8, 0x48dd840, 0xc0246caf00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0029f0270, 0x4bc7440, 0xc01d554000, 0xc01ace4cc0, 0xc0246caf00, 0xc0246caf00, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0029f0270, 0x4bc7440, 0xc01d554000, 0x0, 0xc000100800, 0x0, 0xc013895080)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0029f0270, 0x4bc7440, 0xc01d554000, 0xc01ace4cc0, 0xc0246caec0, 0xc006c8fac8, 0xca72c8, 0x48dd840, 0xc0246caec0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0029f0270, 0x4bc7440, 0xc01d554000, 0xc01ace4cc0, 0xc01ace4d40, 0xc02f07bba0, 0x661046, 0x491f620, 0xc0029f0270)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0029f0270, 0xc013895020, 0xc001681820, 0x5a4e4e0, 0xc0029f0270, 0xc004242c00, 0x330)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00877c700, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00877c700, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00877c700)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217747366 [semacquire, 32 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0114de690, 0x1688c57bbab3b637, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0114de690, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0114de690, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0114de690, 0x4c245c5, 0x10, 0xc0264096c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0114de690, 0xc01a309900, 0xc0114de690, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a6dfa0, 0xc00218c998, 0xc01a309900, 0xc0058cac00, 0xc0114de630, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a6dfa0, 0xc00218c998, 0x4bc7440, 0xc01a309900, 0x21, 0xc026409890, 0x2, 0x2)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0114de5a0, 0x4bc7440, 0xc01a309900, 0xc00b2d0dc0, 0xc00b2d0de0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:137 +0x2dc
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0114de5a0, 0x4bc7440, 0xc01a309900, 0xc026409990, 0xca72c8, 0x48dd840, 0xc012f3bfc0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0114de5a0, 0x4bc7440, 0xc01a309900, 0xc00b2d0dc0, 0xc012f3bfc0, 0xc012f3bfc0, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0114de5a0, 0x4bc7440, 0xc01a309900, 0x0, 0xc00758a000, 0x0, 0xc01dde60c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0114de5a0, 0x4bc7440, 0xc01a309900, 0xc00b2d0dc0, 0xc012f3bf80, 0xc026409ac8, 0xca72c8, 0x48dd840, 0xc012f3bf80)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0114de5a0, 0x4bc7440, 0xc01a309900, 0xc00b2d0dc0, 0xc00b2d0de0, 0xc02eebcba0, 0x661046, 0x491f620, 0xc0114de5a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0114de5a0, 0xc01dde6060, 0xc001681820, 0x5a4e4e0, 0xc0114de5a0, 0xc00387e000, 0x345)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0195cae00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0195cae00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0195cae00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 4875 [IO wait]:
internal/poll.runtime_pollWait(0x7f8b125cb3d0, 0x72, 0x59b54a0)
/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc003166298, 0x72, 0x59b5400, 0x7cf1b20, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc003166280, 0xc00896a000, 0xbe0a, 0xbe0a, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:159 +0x1a5
net.(*netFD).Read(0xc003166280, 0xc00896a000, 0xbe0a, 0xbe0a, 0xc, 0xc, 0xc00896a005)
/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc004ba06a0, 0xc00896a000, 0xbe0a, 0xbe0a, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:182 +0x8e
github.com/cockroachdb/cmux.(*MuxConn).Read(0xc003641440, 0xc00896a000, 0xbe0a, 0xbe0a, 0xc00896a000, 0x5, 0x7f8b3bd0fbe8)
/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x115
crypto/tls.(*atLeastReader).Read(0xc0052e5f20, 0xc00896a000, 0xbe0a, 0xbe0a, 0x1e, 0xbe05, 0xc003c2e798)
/usr/local/go/src/crypto/tls/conn.go:779 +0x62
bytes.(*Buffer).ReadFrom(0xc003494280, 0x59a7c60, 0xc0052e5f20, 0x45f8c5, 0x458c0a0, 0x49151a0)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc003494000, 0x7f8b12553238, 0xc003641440, 0x5, 0xc003641440, 0xd)
/usr/local/go/src/crypto/tls/conn.go:801 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc003494000, 0x0, 0x0, 0xc3213e)
/usr/local/go/src/crypto/tls/conn.go:608 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:576
crypto/tls.(*Conn).Read(0xc003494000, 0xc0035b4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1252 +0x15f
bufio.(*Reader).Read(0xc003780ae0, 0xc0011ce578, 0x9, 0x9, 0xc003597bc0, 0xc0124d6180, 0xc003c2edff)
/usr/local/go/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x59a7a20, 0xc003780ae0, 0xc0011ce578, 0x9, 0x9, 0x9, 0xc003c2edd0, 0x45ae85, 0xc003597bc0)
/usr/local/go/src/io/io.go:314 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:333
golang.org/x/net/http2.readFrameHeader(0xc0011ce578, 0x9, 0x9, 0x59a7a20, 0xc003780ae0, 0x0, 0xc000000000, 0xc002f61ff0, 0x44e0040)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x89
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011ce540, 0xc020d8ebf0, 0x8702740, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa5
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0022b7c80, 0xc002bca120, 0x52fddb8)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0005601a0, 0x5aba220, 0xc0022b7c80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005601a0, 0x5aba220, 0xc0022b7c80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:702 +0x50b
goroutine 217748289 [semacquire, 32 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc002d5ce10, 0x1688c57cf0770034, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc002d5ce10, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc002d5ce10, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc002d5ce10, 0x4c245c5, 0x10, 0xc0059b56c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc002d5ce10, 0xc01f772a00, 0xc002d5ce10, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a6dfa0, 0xc0010914f8, 0xc01f772a00, 0xc0058cac00, 0xc002d5cc30, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a6dfa0, 0xc0010914f8, 0x4bc7440, 0xc01f772a00, 0x21, 0xc0059b5890, 0x2, 0x2)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc002d5c8a0, 0x4bc7440, 0xc01f772a00, 0xc002f73920, 0xc002f73940, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:137 +0x2dc
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc002d5c8a0, 0x4bc7440, 0xc01f772a00, 0xc0059b5990, 0xca72c8, 0x48dd840, 0xc00dab6140)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc002d5c8a0, 0x4bc7440, 0xc01f772a00, 0xc002f73920, 0xc00dab6140, 0xc00dab6140, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc002d5c8a0, 0x4bc7440, 0xc01f772a00, 0x0, 0xc00fba5800, 0x0, 0xc012771680)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc002d5c8a0, 0x4bc7440, 0xc01f772a00, 0xc002f73920, 0xc00dab6100, 0xc0059b5ac8, 0xca72c8, 0x48dd840, 0xc00dab6100)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc002d5c8a0, 0x4bc7440, 0xc01f772a00, 0xc002f73920, 0xc002f73940, 0xc0037e2ba0, 0x661046, 0x491f620, 0xc002d5c8a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc002d5c8a0, 0xc012771620, 0xc001681820, 0x5a4e4e0, 0xc002d5c8a0, 0xc00d49d800, 0x340)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0107f5800, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0107f5800, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0107f5800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 6647 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595f4, 0x1688bcc052c0cf00, 0x1)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*Mutex).lockSlow(0xc019f595f0)
/usr/local/go/src/sync/mutex.go:138 +0x105
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0xc019f595f0)
/usr/local/go/src/sync/rwmutex.go:98 +0x90
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).BumpSideTransportClosed(0xc019f59200, 0x5a4e4e0, 0xc0018e6b10, 0x1688bcc052c0cff4, 0x0, 0x1688bcbf9ff071f4, 0x0, 0x1688bcc0826fd7f4, 0x100000000, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_closedts.go:85 +0xa5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Sender).publish(0xc000ae12b0, 0x5a4e4e0, 0xc0018e6b10, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:385 +0x9f5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Sender).Run.func2(0x5a4e4e0, 0xc0018e6b10)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:262 +0x245
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc0018e6b10, 0xc000a70b40, 0xc009daede0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 216096340 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc010eedd40, 0x1688bcc53123bdce, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc010eedd40, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc010eedd10, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc010eedd10, 0x4c245c5, 0x10, 0xc0033cd6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc010eedd10, 0xc0142fc500, 0xc010eedd10, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc010eedcb0, 0xc0142fc500, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc010eedcb0, 0x4bc7440, 0xc0142fc500, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc010eedcb0, 0x4bc7440, 0xc0142fc500, 0xc0148551c0, 0xc0148551e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc010eedcb0, 0x4bc7440, 0xc0142fc500, 0xc0033cd990, 0xca72c8, 0x48dd840, 0xc0126c89c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc010eedcb0, 0x4bc7440, 0xc0142fc500, 0xc0148551c0, 0xc0126c89c0, 0xc0126c89c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc010eedcb0, 0x4bc7440, 0xc0142fc500, 0x0, 0xc024c79800, 0x0, 0xc024d8ea80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc010eedcb0, 0x4bc7440, 0xc0142fc500, 0xc0148551c0, 0xc0126c8980, 0xc0033cdac8, 0xca72c8, 0x48dd840, 0xc0126c8980)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc010eedcb0, 0x4bc7440, 0xc0142fc500, 0xc0148551c0, 0xc0148551e0, 0xc002294ba0, 0x661046, 0x491f620, 0xc010eedcb0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc010eedcb0, 0xc024d8ea20, 0xc001681820, 0x5a4e4e0, 0xc010eedcb0, 0xc00dd57800, 0x336)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01466ec00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01466ec00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01466ec00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 4874 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0022b7c80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x28a
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10fb
goroutine 216311469 [semacquire, 174 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01d643440, 0x1688bdbaa5f791ee, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01d643440, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01d643410, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01d643410, 0x4c245c5, 0x10, 0xc005a616c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01d643410, 0xc001644960, 0xc01d643410, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01d6433b0, 0xc001644960, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01d6433b0, 0x4bc7440, 0xc001644960, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01d6433b0, 0x4bc7440, 0xc001644960, 0xc020e6cf40, 0xc020e6cf60, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01d6433b0, 0x4bc7440, 0xc001644960, 0xc005a61990, 0xca72c8, 0x48dd840, 0xc0125570c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01d6433b0, 0x4bc7440, 0xc001644960, 0xc020e6cf40, 0xc0125570c0, 0xc0125570c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01d6433b0, 0x4bc7440, 0xc001644960, 0x0, 0xc000100800, 0x0, 0xc0065ceae0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01d6433b0, 0x4bc7440, 0xc001644960, 0xc020e6cf40, 0xc012556b00, 0xc005a61ac8, 0xca72c8, 0x48dd840, 0xc012556b00)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01d6433b0, 0x4bc7440, 0xc001644960, 0xc020e6cf40, 0xc020e6cf60, 0xc0020f6ba0, 0x661046, 0x491f620, 0xc01d6433b0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01d6433b0, 0xc0065cea80, 0xc001681820, 0x5a4e4e0, 0xc01d6433b0, 0xc00a53b200, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01264fe00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01264fe00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01264fe00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217868515 [semacquire, 23 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc004293590, 0x1688c5ff5c44cf3c, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc004293590, 0x1688c5ff5c44cf3c, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc004293530, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc004293530, 0x4c245c5, 0x10, 0xc008fa96c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc004293530, 0xc0100a75e0, 0xc004293530, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc004293020, 0xc0100a75e0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc004293020, 0x4bc7440, 0xc0100a75e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc004293020, 0x4bc7440, 0xc0100a75e0, 0xc000793c20, 0xc000793c40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc004293020, 0x4bc7440, 0xc0100a75e0, 0xc008fa9990, 0xca72c8, 0x48dd840, 0xc0104248c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc004293020, 0x4bc7440, 0xc0100a75e0, 0xc000793c20, 0xc0104248c0, 0xc0104248c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc004293020, 0x4bc7440, 0xc0100a75e0, 0x0, 0xc003d76000, 0x0, 0xc02853e000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc004293020, 0x4bc7440, 0xc0100a75e0, 0xc000793c20, 0xc010424880, 0xc008fa9ac8, 0xca72c8, 0x48dd840, 0xc010424880)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc004293020, 0x4bc7440, 0xc0100a75e0, 0xc000793c20, 0xc000793c40, 0xc003019ba0, 0x661046, 0x491f620, 0xc004293020)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc004293020, 0xc014417f20, 0xc001681820, 0x5a4e4e0, 0xc004293020, 0xc0065ef000, 0x425)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0042b3c00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0042b3c00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0042b3c00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 5024 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc003137220, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x125
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0035c0300, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1d3
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc003d78300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xd7
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10d6
goroutine 217919809 [semacquire, 18 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0103b8180, 0x1688c644f9e59e01, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0103b8180, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0103b8150, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0103b8150, 0x4c245c5, 0x10, 0xc019bed6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0103b8150, 0xc024af74a0, 0xc0103b8150, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0103b80f0, 0xc024af74a0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0103b80f0, 0x4bc7440, 0xc024af74a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0103b80f0, 0x4bc7440, 0xc024af74a0, 0xc00f355340, 0xc00f355360, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0103b80f0, 0x4bc7440, 0xc024af74a0, 0xc019bed990, 0xca72c8, 0x48dd840, 0xc02016d140)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0103b80f0, 0x4bc7440, 0xc024af74a0, 0xc00f355340, 0xc02016d140, 0xc02016d140, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0103b80f0, 0x4bc7440, 0xc024af74a0, 0x0, 0xc00fba5800, 0x0, 0xc020c552c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0103b80f0, 0x4bc7440, 0xc024af74a0, 0xc00f355340, 0xc02016d100, 0xc019bedac8, 0xca72c8, 0x48dd840, 0xc02016d100)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0103b80f0, 0x4bc7440, 0xc024af74a0, 0xc00f355340, 0xc00f355360, 0xc00301dba0, 0x661046, 0x491f620, 0xc0103b80f0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0103b80f0, 0xc020c55260, 0xc001681820, 0x5a4e4e0, 0xc0103b80f0, 0xc0198e3200, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00a035600, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00a035600, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00a035600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216290438 [semacquire, 177 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc015bd6030, 0x1688bd9d8b4366f9, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc015bd6030, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc015bd6000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc015bd6000, 0x4c245c5, 0x10, 0xc0048f16c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc015bd6000, 0xc021552140, 0xc015bd6000, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc002a15860, 0xc021552140, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc002a15860, 0x4bc7440, 0xc021552140, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc002a15860, 0x4bc7440, 0xc021552140, 0xc01a89fe80, 0xc01a89fea0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc002a15860, 0x4bc7440, 0xc021552140, 0xc0048f1990, 0xca72c8, 0x48dd840, 0xc00e1eaa40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc002a15860, 0x4bc7440, 0xc021552140, 0xc01a89fe80, 0xc00e1eaa40, 0xc00e1eaa40, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc002a15860, 0x4bc7440, 0xc021552140, 0x0, 0xc003d76000, 0x0, 0xc00f6e07e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc002a15860, 0x4bc7440, 0xc021552140, 0xc01a89fe80, 0xc00e1eaa00, 0xc0048f1ac8, 0xca72c8, 0x48dd840, 0xc00e1eaa00)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc002a15860, 0x4bc7440, 0xc021552140, 0xc01a89fe80, 0xc01a89fea0, 0xc004b4aba0, 0x661046, 0x491f620, 0xc002a15860)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc002a15860, 0xc00f6e0780, 0xc001681820, 0x5a4e4e0, 0xc002a15860, 0xc0099cd800, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0057fde00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0057fde00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0057fde00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216085876 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01b434ae0, 0x1688bcc0546f02c4, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01b434ae0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01b434ae0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01b434ae0, 0x4c245c5, 0x10, 0xc020862410, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01b434ae0, 0xc00188b540, 0xc01b434ae0, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01b434ab0, 0xc00188b540, 0xc021bbbb00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc01aca9500, 0x5a4e4e0, 0xc01b434ab0, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc01aca9500, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc006d0bad0, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc006d0ba98, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc006d0ba68, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc006d0b9c8, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc006d0b9c8, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc006d0b8f8, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc006d0b8d8, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc006d0b840, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc006d0b680, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00f2b6cf0, 0x5a4e420, 0xc021bbbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc021bbbb00, 0xc020867208, 0xc00a3b8000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc00f2b6cf0, 0x5a4e420, 0xc021bbbb00, 0xc00a3b8000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc00f2b6cf0, 0x5a4e420, 0xc021bbbb00, 0xc00a3b8000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc011959228, 0x5a4e420, 0xc021bbbb00, 0xc00e7f7500, 0xc0208964e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc011959200, 0x5a4e420, 0xc021bbbb00, 0xc00e7f7500, 0xc0208964e0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc015f67e60, 0x5a4e420, 0xc021bbbb00, 0xc00e7f7500, 0xc0208964e0, 0xc020867400, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc015f67e60, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc0208675a0, 0x4bfc8f0, 0x5, 0x5a50860, 0xc015f67e60)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc0208675a0, 0x5a50860, 0xc015f67e60, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc0208675a0, 0x5a50860, 0xc015f67e60, 0x0, 0xc0113bc0a0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc021bbbb00, 0xc00e7f7500, 0xc0208964e0, 0x5a50860, 0xc015f67e60, 0x5a4e420, 0xc021bbbb00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc012d4f800, 0x5a4e420, 0xc021bbbb00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc00dbbfb00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc020867768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc014121200, 0x5a4e420, 0xc021bbbb00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc014121200, 0x5a4e420, 0xc021bbbb00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc013763560, 0x5a4e420, 0xc021bbbb00, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc0091160e0, 0xc00f2b6cf0, 0xc004b1e480, 0xc00bee6700, 0xc0208966f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc019149dd0, 0xc0208966f8, 0xc0091160e0, 0xc00f2b6cf0, 0x5a50860, 0xc015f67e60, 0x0, 0xc00bee6700, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc020896000, 0x5a4e4e0, 0xc019149dd0, 0xc0208964e0, 0x2, 0x7f8b0dd7dcb0, 0xc00138ca80, 0xc020868000, 0xc009360298, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc020896000, 0x5a4e4e0, 0xc019149dd0, 0xc0208964e0, 0x7f8b0dd7dcb0, 0xc00138ca80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc020896000, 0x5a4e4e0, 0xc019149dd0, 0x5a6dda0, 0xc004e84300, 0xc006c21035, 0x1e9, 0x11, 0x1, 0xc0132163c0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc020896000, 0x5a4e420, 0xc011485200, 0x5a6dda0, 0xc004e84300, 0xc006c21035, 0x1e9, 0x11, 0x1, 0xc0132163c0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc020896000, 0x5a4e420, 0xc011485200, 0xc0132163c0, 0xc013762d80, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc006c219fd, 0x0, 0x0, 0x35d79cb1, 0xed85a80bf, 0x0, 0xc020896000, 0xc020869a20, 0xc020869a10, 0x17, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc020896000, 0x5a4e420, 0xc011485200, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc020896000, 0x5a4e420, 0xc0195f9040, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc01da7bd90, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc0195f9040, 0xc020896000, 0x5400, 0x15000, 0xc001805a40, 0xc01da7bd90, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc00ddc5b41, 0xc007b15900, 0x5a4e420, 0xc0195f9040, 0xc01da7bd90, 0xc001a38000, 0xc00138c400, 0x5ab5540, 0xc009a03680, 0xc0162a8060, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216090717 [chan receive, 105 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc01fcb5b00, 0x5a4e4e0, 0xc020588300, 0xc020c4ffe0, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc020588300, 0x5ab2120, 0xc020718000, 0xc020585028, 0xd, 0xc0205882d0, 0x59fb820, 0xc0202c6870, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc020588300, 0x5ab21e0, 0xc015cd1c80, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc015cd1c80)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc015cd1c80, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 10070 [sync.Cond.Wait, 192 minutes]:
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:312
sync.runtime_notifyListWait(0xc0002f7ba8, 0xc000252e3d)
/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0002f7b98)
/usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc0002f7b90, 0x5a4e420, 0xc006df6fc0, 0x129756, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:602 +0xa5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1(0x5a4e4e0, 0xc0018e6b10)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:802 +0x3dc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc0018e6b10, 0xc000a70b40, 0xc00367aea0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 217433874 [semacquire, 65 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc006463740, 0x1688c3afb9999682, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc006463740, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc015f37da0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc015f37da0, 0x4c245c5, 0x10, 0xc00885b6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc015f37da0, 0xc00fecee60, 0xc015f37da0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc015f37b00, 0xc00fecee60, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc015f37b00, 0x4bc7440, 0xc00fecee60, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc015f37b00, 0x4bc7440, 0xc00fecee60, 0xc013b59720, 0xc013b59740, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc015f37b00, 0x4bc7440, 0xc00fecee60, 0xc00885b990, 0xca72c8, 0x48dd840, 0xc012b8bc00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc015f37b00, 0x4bc7440, 0xc00fecee60, 0xc013b59720, 0xc012b8bc00, 0xc012b8bc00, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc015f37b00, 0x4bc7440, 0xc00fecee60, 0x0, 0xc000100400, 0x0, 0xc015bfac60)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc015f37b00, 0x4bc7440, 0xc00fecee60, 0xc013b59720, 0xc012b8bbc0, 0xc00885bac8, 0xca72c8, 0x48dd840, 0xc012b8bbc0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc015f37b00, 0x4bc7440, 0xc00fecee60, 0xc013b59720, 0xc013b59740, 0xc007304ba0, 0x661046, 0x491f620, 0xc015f37b00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc015f37b00, 0xc015bfac00, 0xc001681820, 0x5a4e4e0, 0xc015f37b00, 0xc00ba77800, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0164ea600, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0164ea600, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0164ea600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217748182 [semacquire, 32 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc020cce960, 0x1688c57cbcf78bcd, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc020cce960, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc020cce8d0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc020cce8d0, 0x4c245c5, 0x10, 0xc008ac36c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc020cce8d0, 0xc010835220, 0xc020cce8d0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc020cce840, 0xc010835220, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc020cce840, 0x4bc7440, 0xc010835220, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc020cce840, 0x4bc7440, 0xc010835220, 0xc00dfe8be0, 0xc00dfe8c00, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc020cce840, 0x4bc7440, 0xc010835220, 0xc008ac3990, 0xca72c8, 0x48dd840, 0xc0218ce9c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc020cce840, 0x4bc7440, 0xc010835220, 0xc00dfe8be0, 0xc0218ce9c0, 0xc0218ce9c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc020cce840, 0x4bc7440, 0xc010835220, 0x0, 0xc000100400, 0x0, 0xc00d2e87e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc020cce840, 0x4bc7440, 0xc010835220, 0xc00dfe8be0, 0xc0218ce940, 0xc008ac3ac8, 0xca72c8, 0x48dd840, 0xc0218ce940)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc020cce840, 0x4bc7440, 0xc010835220, 0xc00dfe8be0, 0xc00dfe8c00, 0xc0072a7ba0, 0x661046, 0x491f620, 0xc020cce840)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc020cce840, 0xc00d2e86c0, 0xc001681820, 0x5a4e4e0, 0xc020cce840, 0xc0048f3200, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00b7a4200, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00b7a4200, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00b7a4200)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217674499 [semacquire, 42 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc002fbe100, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc002fbe1e0, 0xc0066c1388)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:372 +0x1db
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).ComputeStatsForKeySpan(0xc0018ad000, 0xc002d1c1d0, 0x5, 0x8, 0xc002d1c1d8, 0x4, 0x8, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:2710 +0x15f
github.com/cockroachdb/cockroach/pkg/server.(*statusServer).SpanStats.func1(0xc0018ad000, 0x464925, 0xc00301b658)
/go/src/github.com/cockroachdb/cockroach/pkg/server/status.go:2273 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores.func1(0x1, 0xc0018ad000, 0xc00301b658)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:148 +0x38
github.com/cockroachdb/cockroach/pkg/util/syncutil.(*IntMap).Range(0xc0002f7c30, 0xc0066c16e8)
/go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/int_map.go:352 +0x130
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores(0xc0002f7c00, 0xc00301b768, 0x1, 0x100000001)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:147 +0x75
github.com/cockroachdb/cockroach/pkg/server.(*statusServer).SpanStats(0xc00589e840, 0x5a4e4e0, 0xc002fbe1b0, 0xc01b2d9340, 0xc00589e840, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/status.go:2272 +0x170
github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_SpanStats_Handler.func1(0x5a4e4e0, 0xc002fbe0f0, 0x4a19420, 0xc01b2d9340, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:4761 +0x8b
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc002fbe0f0, 0x4a19420, 0xc01b2d9340, 0xc019227260, 0xc019227280, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc002fbe0f0, 0x4a19420, 0xc01b2d9340, 0xc00301b990, 0xca72c8, 0x48dd840, 0xc01b2d93c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc002fbe0f0, 0x4a19420, 0xc01b2d9340, 0xc019227260, 0xc01b2d93c0, 0xc01b2d93c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc002fbe0f0, 0x4a19420, 0xc01b2d9340, 0x0, 0xc000100800, 0x0, 0xc008f17260)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc002fbe0f0, 0x4a19420, 0xc01b2d9340, 0xc019227260, 0xc01b2d9380, 0xc00301bac8, 0xca72c8, 0x48dd840, 0xc01b2d9380)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc002fbe0f0, 0x4a19420, 0xc01b2d9340, 0xc019227260, 0xc019227280, 0xc00301bba0, 0x661046, 0x491f620, 0xc002fbe0f0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_SpanStats_Handler(0x4bcfa40, 0xc00589e840, 0x5a4e4e0, 0xc002fbe0f0, 0xc008f171a0, 0xc001681820, 0x5a4e4e0, 0xc002fbe0f0, 0xc020fda600, 0xf)
/go/src/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:4763 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0144d2a00, 0xc00076a8a0, 0x7cd3c80, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0144d2a00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0144d2a00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218082092 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02a81ae10, 0x1688c72a2c15004e, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02a81ae10, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02a81ade0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02a81ade0, 0x4c245c5, 0x10, 0xc02afcf6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02a81ade0, 0xc02a307c20, 0xc02a81ade0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02a81ad50, 0xc02a307c20, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02a81ad50, 0x4bc7440, 0xc02a307c20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02a81ad50, 0x4bc7440, 0xc02a307c20, 0xc01a3af3c0, 0xc01a3af3e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02a81ad50, 0x4bc7440, 0xc02a307c20, 0xc02afcf990, 0xca72c8, 0x48dd840, 0xc016d80380)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02a81ad50, 0x4bc7440, 0xc02a307c20, 0xc01a3af3c0, 0xc016d80380, 0xc016d80380, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02a81ad50, 0x4bc7440, 0xc02a307c20, 0x0, 0xc003d76000, 0x0, 0xc02a351560)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02a81ad50, 0x4bc7440, 0xc02a307c20, 0xc01a3af3c0, 0xc016d80340, 0xc02afcfac8, 0xca72c8, 0x48dd840, 0xc016d80340)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02a81ad50, 0x4bc7440, 0xc02a307c20, 0xc01a3af3c0, 0xc01a3af3e0, 0xc004b46ba0, 0x661046, 0x491f620, 0xc02a81ad50)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02a81ad50, 0xc02a351500, 0xc001681820, 0x5a4e4e0, 0xc02a81ad50, 0xc02a844000, 0x2e9)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc02a832100, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc02a832100, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc02a832100)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216392203 [semacquire, 167 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc025851f80, 0x1688be1fee72c771, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc025851f80, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc025851f20, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc025851f20, 0x4c245c5, 0x10, 0xc006a776c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc025851f20, 0xc002631720, 0xc025851f20, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc025851e90, 0xc002631720, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc025851e90, 0x4bc7440, 0xc002631720, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc025851e90, 0x4bc7440, 0xc002631720, 0xc0114cf020, 0xc0114cf040, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc025851e90, 0x4bc7440, 0xc002631720, 0xc006a77990, 0xca72c8, 0x48dd840, 0xc01e6380c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc025851e90, 0x4bc7440, 0xc002631720, 0xc0114cf020, 0xc01e6380c0, 0xc01e6380c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc025851e90, 0x4bc7440, 0xc002631720, 0x0, 0xc00758a000, 0x0, 0xc0151ec360)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc025851e90, 0x4bc7440, 0xc002631720, 0xc0114cf020, 0xc01e638080, 0xc006a77ac8, 0xca72c8, 0x48dd840, 0xc01e638080)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc025851e90, 0x4bc7440, 0xc002631720, 0xc0114cf020, 0xc0114cf040, 0xc003a36ba0, 0x661046, 0x491f620, 0xc025851e90)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc025851e90, 0xc0151ec300, 0xc001681820, 0x5a4e4e0, 0xc025851e90, 0xc00d110c00, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc005afa700, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc005afa700, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc005afa700)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217758186 [semacquire, 31 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01e33d4d0, 0x1688c58ab54b00a5, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01e33d4d0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01e33d0b0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01e33d0b0, 0x4c245c5, 0x10, 0xc0072336c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01e33d0b0, 0xc013ecb040, 0xc01e33d0b0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01e33d050, 0xc013ecb040, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01e33d050, 0x4bc7440, 0xc013ecb040, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01e33d050, 0x4bc7440, 0xc013ecb040, 0xc016231100, 0xc016231120, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01e33d050, 0x4bc7440, 0xc013ecb040, 0xc007233990, 0xca72c8, 0x48dd840, 0xc00b357000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01e33d050, 0x4bc7440, 0xc013ecb040, 0xc016231100, 0xc00b357000, 0xc00b357000, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01e33d050, 0x4bc7440, 0xc013ecb040, 0x0, 0xc0002a9800, 0x0, 0xc0197eb7a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01e33d050, 0x4bc7440, 0xc013ecb040, 0xc016231100, 0xc00b356fc0, 0xc007233ac8, 0xca72c8, 0x48dd840, 0xc00b356fc0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01e33d050, 0x4bc7440, 0xc013ecb040, 0xc016231100, 0xc016231120, 0xc0020f3ba0, 0x661046, 0x491f620, 0xc01e33d050)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01e33d050, 0xc0197eb740, 0xc001681820, 0x5a4e4e0, 0xc01e33d050, 0xc00a608c00, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0107f5400, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0107f5400, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0107f5400)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216094752 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc009a16870, 0x1688bcc4734836d7, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc009a16870, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc009a16840, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc009a16840, 0x4c245c5, 0x10, 0xc0051296c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc009a16840, 0xc0255dc000, 0xc009a16840, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc009a167e0, 0xc0255dc000, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc009a167e0, 0x4bc7440, 0xc0255dc000, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc009a167e0, 0x4bc7440, 0xc0255dc000, 0xc019ed0520, 0xc019ed0540, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc009a167e0, 0x4bc7440, 0xc0255dc000, 0xc005129990, 0xca72c8, 0x48dd840, 0xc01af6b480)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc009a167e0, 0x4bc7440, 0xc0255dc000, 0xc019ed0520, 0xc01af6b480, 0xc01af6b480, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc009a167e0, 0x4bc7440, 0xc0255dc000, 0x0, 0xc003d76000, 0x0, 0xc0091b50e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc009a167e0, 0x4bc7440, 0xc0255dc000, 0xc019ed0520, 0xc01af6b440, 0xc005129ac8, 0xca72c8, 0x48dd840, 0xc01af6b440)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc009a167e0, 0x4bc7440, 0xc0255dc000, 0xc019ed0520, 0xc019ed0540, 0xc0072a2ba0, 0x661046, 0x491f620, 0xc009a167e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc009a167e0, 0xc0091b5080, 0xc001681820, 0x5a4e4e0, 0xc009a167e0, 0xc006cc9800, 0x34b)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc015192600, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc015192600, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc015192600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217020188 [semacquire, 103 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc021d1b9e0, 0x1688c1a756975e7f, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc021d1b9e0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc021d1b9b0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc021d1b9b0, 0x4c245c5, 0x10, 0xc002f9d6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc021d1b9b0, 0xc01d57e460, 0xc021d1b9b0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc021d1b950, 0xc01d57e460, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc021d1b950, 0x4bc7440, 0xc01d57e460, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc021d1b950, 0x4bc7440, 0xc01d57e460, 0xc01aff0fc0, 0xc01aff0fe0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc021d1b950, 0x4bc7440, 0xc01d57e460, 0xc002f9d990, 0xca72c8, 0x48dd840, 0xc010fd9980)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc021d1b950, 0x4bc7440, 0xc01d57e460, 0xc01aff0fc0, 0xc010fd9980, 0xc010fd9980, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc021d1b950, 0x4bc7440, 0xc01d57e460, 0x0, 0xc00009c800, 0x0, 0x7f8b1252aa18)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc021d1b950, 0x4bc7440, 0xc01d57e460, 0xc01aff0fc0, 0xc010fd9940, 0xc002f9dac8, 0xca72c8, 0x48dd840, 0xc010fd9940)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc021d1b950, 0x4bc7440, 0xc01d57e460, 0xc01aff0fc0, 0xc01aff0fe0, 0xc001ea7ba0, 0x661046, 0x491f620, 0xc021d1b950)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc021d1b950, 0xc01d7d32c0, 0xc001681820, 0x5a4e4e0, 0xc021d1b950, 0xc005bcb800, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00bdfa800, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00bdfa800, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00bdfa800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 10069 [sync.Cond.Wait, 192 minutes]:
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:312
sync.runtime_notifyListWait(0xc0002f7ba8, 0xc000252e3c)
/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc0002f7b98)
/usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc0002f7b90, 0x5a4e420, 0xc005397500, 0x129756, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:602 +0xa5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1(0x5a4e4e0, 0xc0018e6b10)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:802 +0x3dc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000c04b80, 0x5a4e4e0, 0xc0018e6b10, 0xc000a70b40, 0xc00367ae80)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:351 +0xb9
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xfc
goroutine 216997583 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01e8bc720, 0x1688c18c6ba947d0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01e8bc720, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01e8bc6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01e8bc6c0, 0x4c245c5, 0x10, 0xc0057d96c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01e8bc6c0, 0xc006816f00, 0xc01e8bc6c0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01e8bc660, 0xc006816f00, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01e8bc660, 0x4bc7440, 0xc006816f00, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01e8bc660, 0x4bc7440, 0xc006816f00, 0xc00f9f7d40, 0xc00f9f7d60, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01e8bc660, 0x4bc7440, 0xc006816f00, 0xc0057d9990, 0xca72c8, 0x48dd840, 0xc019ab0d80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01e8bc660, 0x4bc7440, 0xc006816f00, 0xc00f9f7d40, 0xc019ab0d80, 0xc019ab0d80, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01e8bc660, 0x4bc7440, 0xc006816f00, 0x0, 0xc00009c800, 0x0, 0xc00801ea80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01e8bc660, 0x4bc7440, 0xc006816f00, 0xc00f9f7d40, 0xc019ab0d40, 0xc0057d9ac8, 0xca72c8, 0x48dd840, 0xc019ab0d40)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01e8bc660, 0x4bc7440, 0xc006816f00, 0xc00f9f7d40, 0xc00f9f7d60, 0xc003c28ba0, 0x661046, 0x491f620, 0xc01e8bc660)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01e8bc660, 0xc00801ea20, 0xc001681820, 0x5a4e4e0, 0xc01e8bc660, 0xc00c6c9200, 0x335)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00e711a00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00e711a00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00e711a00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218080042 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc029f3d890, 0x1688c727ff1de72e, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc029f3d890, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc029f3d860, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc029f3d860, 0x4c245c5, 0x10, 0xc01f3496c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc029f3d860, 0xc0258d4d20, 0xc029f3d860, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc029f3d7d0, 0xc0258d4d20, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc029f3d7d0, 0x4bc7440, 0xc0258d4d20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc029f3d7d0, 0x4bc7440, 0xc0258d4d20, 0xc0213c3c80, 0xc0213c3ca0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc029f3d7d0, 0x4bc7440, 0xc0258d4d20, 0xc01f349990, 0xca72c8, 0x48dd840, 0xc0126ec400)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc029f3d7d0, 0x4bc7440, 0xc0258d4d20, 0xc0213c3c80, 0xc0126ec400, 0xc0126ec400, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc029f3d7d0, 0x4bc7440, 0xc0258d4d20, 0x0, 0xc003d76000, 0x0, 0xc011d3c1e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc029f3d7d0, 0x4bc7440, 0xc0258d4d20, 0xc0213c3c80, 0xc0126ec3c0, 0xc01f349ac8, 0xca72c8, 0x48dd840, 0xc0126ec3c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc029f3d7d0, 0x4bc7440, 0xc0258d4d20, 0xc0213c3c80, 0xc0213c3ca0, 0xc003a32ba0, 0x661046, 0x491f620, 0xc029f3d7d0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc029f3d7d0, 0xc011d3c180, 0xc001681820, 0x5a4e4e0, 0xc029f3d7d0, 0xc01eb04600, 0x32b)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029476000, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029476000, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029476000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216997567 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc008879c20, 0x1688c18c62e45e30, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc008879c20, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc008879bf0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc008879bf0, 0x4c245c5, 0x10, 0xc004f896c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc008879bf0, 0xc010c4b5e0, 0xc008879bf0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc008879b90, 0xc010c4b5e0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc008879b90, 0x4bc7440, 0xc010c4b5e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc008879b90, 0x4bc7440, 0xc010c4b5e0, 0xc01da7c2a0, 0xc01da7c2c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc008879b90, 0x4bc7440, 0xc010c4b5e0, 0xc004f89990, 0xca72c8, 0x48dd840, 0xc005ac2bc0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc008879b90, 0x4bc7440, 0xc010c4b5e0, 0xc01da7c2a0, 0xc005ac2bc0, 0xc005ac2bc0, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc008879b90, 0x4bc7440, 0xc010c4b5e0, 0x0, 0xc00fba5800, 0x0, 0xc016c1b440)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc008879b90, 0x4bc7440, 0xc010c4b5e0, 0xc01da7c2a0, 0xc005ac2b80, 0xc004f89ac8, 0xca72c8, 0x48dd840, 0xc005ac2b80)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc008879b90, 0x4bc7440, 0xc010c4b5e0, 0xc01da7c2a0, 0xc01da7c2c0, 0xc02eb7bba0, 0x661046, 0x491f620, 0xc008879b90)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc008879b90, 0xc016c1b3e0, 0xc001681820, 0x5a4e4e0, 0xc008879b90, 0xc005b5e000, 0x326)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00bf97700, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00bf97700, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00bf97700)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216088898 [chan receive, 105 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc008dda000, 0x5a4e4e0, 0xc0107a8a50, 0xc00998aa90, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc0107a8a50, 0x5ab2120, 0xc00bfef180, 0xc008dd803b, 0xd, 0xc0107a8a20, 0x59fb820, 0xc0159d5b00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc0107a8a50, 0x5ab21e0, 0xc00ca1f920, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc00ca1f920)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc00ca1f920, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 218082383 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02dbc62a0, 0x1688c72a519f5f80, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02dbc62a0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02dbc6270, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02dbc6270, 0x4c245c5, 0x10, 0xc02dd136c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02dbc6270, 0xc02dc5f7c0, 0xc02dbc6270, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02dbc6210, 0xc02dc5f7c0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02dbc6210, 0x4bc7440, 0xc02dc5f7c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02dbc6210, 0x4bc7440, 0xc02dc5f7c0, 0xc00fd0cce0, 0xc00fd0cd00, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02dbc6210, 0x4bc7440, 0xc02dc5f7c0, 0xc02dd13990, 0xca72c8, 0x48dd840, 0xc0111a8e80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02dbc6210, 0x4bc7440, 0xc02dc5f7c0, 0xc00fd0cce0, 0xc0111a8e80, 0xc0111a8e80, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02dbc6210, 0x4bc7440, 0xc02dc5f7c0, 0x0, 0xc003d76000, 0x0, 0xc02dbc4300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02dbc6210, 0x4bc7440, 0xc02dc5f7c0, 0xc00fd0cce0, 0xc0111a8e40, 0xc02dd13ac8, 0xca72c8, 0x48dd840, 0xc0111a8e40)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02dbc6210, 0x4bc7440, 0xc02dc5f7c0, 0xc00fd0cce0, 0xc00fd0cd00, 0xc004b46ba0, 0x661046, 0x491f620, 0xc02dbc6210)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02dbc6210, 0xc02dbc42a0, 0xc001681820, 0x5a4e4e0, 0xc02dbc6210, 0xc02dc9a000, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02da6d000, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02da6d000, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02da6d000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 5025 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc003d78300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x28a
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10fb
goroutine 216996062 [semacquire, 105 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc009289838, 0x1688c1889457a894, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc009289838, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00672db00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00672db00, 0x4c245c5, 0x10, 0xc02a3c25e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00672db00, 0xc021099e00, 0xc00672db00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00672dad0, 0xc021099e00, 0xc009289698, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc011abbb80, 0x5a4e4e0, 0xc00672dad0, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc011abbb80, 0x5a6dfa0, 0xc009289698, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc009289698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc009289698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc009289698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1145 +0x3e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc0092894f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:784 +0x5e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc005449650, 0x5a6dfa0, 0xc0092894f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc005449618, 0x5a6dfa0, 0xc0092894f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc0054495e8, 0x5a6dfa0, 0xc0092894f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:126 +0x8db
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc005449548, 0x5a6dfa0, 0xc0092894f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc005449548, 0x5a6dfa0, 0xc0092894f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc005449478, 0x5a6dfa0, 0xc0092894f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc005449458, 0x5a6dfa0, 0xc0092894f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0054493c0, 0x5a6dfa0, 0xc0092894f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc005449200, 0x5a6dfa0, 0xc009289358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a6dfa0, 0xc009289358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc010015d40, 0x5a6dfa0, 0xc009289358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcher.func1(0x5a6dfa0, 0xc009289358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:217 +0x85
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc008889260, 0x5a6dfa0, 0xc009289358, 0x485e900, 0x401)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:379 +0x739
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc008889260, 0x5a6dfa0, 0xc009289358, 0x292dbf7, 0xc008337540, 0x28, 0x28, 0x400, 0x59a93e0, 0xc006418510, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:480 +0x1ab
github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc00caeccf0, 0x5a6dfa0, 0xc009289358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:129 +0x104
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).nextBatch(0xc005316960, 0x5a6dfa0, 0xc009289358, 0xc01ac04580, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:803 +0xb9
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:785
github.com/cockroachdb/cockroach/pkg/sql/colmem.(*Allocator).PerformOperation(0xc015bd63c0, 0x0, 0x0, 0x0, 0xc02a3c7910)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colmem/allocator.go:300 +0x59
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc005316960, 0x5a6dfa0, 0xc009289358, 0xc, 0x5a6dfa0, 0xc009289358, 0xc0092891e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:777 +0xa5
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc005a59600, 0x5a6dfa0, 0xc009289018, 0x3b249650047976e, 0x60c89e54)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x87
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc00672c030, 0x5a6dfa0, 0xc009289018, 0xc02a3c7a38, 0xa9fbc6)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:48 +0x69
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc001f18370, 0x5a6dfa0, 0xc009289018, 0xa, 0x46cb65)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).Next(0xc01f285800, 0x5a6dfa0, 0xc009289018, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/stats.go:86 +0x5d
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selEQInt64Int64ConstOp).Next(0xc00caec2d0, 0x5a6dfa0, 0xc009289018, 0xc022209fc0, 0xc005a59600)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:6114 +0xf7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selGEInt64Int64ConstOp).Next(0xc00caec360, 0x5a6dfa0, 0xc009289018, 0x2, 0x20)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:54994 +0xf7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selGTTimestampTimestampConstOp).Next(0xc021098780, 0x5a6dfa0, 0xc009289018, 0xc00c57bff8, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:48146 +0x237
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc01e33fbc0, 0x5a6dfa0, 0xc009289018, 0x0, 0x1003000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/is_null_ops.eg.go:241 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BatchSchemaSubsetEnforcer).Next(0xc01eb595c0, 0x5a6dfa0, 0xc009289018, 0xc0058c7040, 0x52ffda8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:212 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*bufferOp).advance(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/buffer.go:62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*caseOp).Next(0xc00caec750, 0x5a6dfa0, 0xc009289018, 0x0, 0x7f8b10ee8268)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/case.go:109 +0x6b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.selBoolOp.Next(0x5a518a0, 0xc00caec750, 0x0, 0x0, 0xc01910dec0, 0xf, 0x5a6dfa0, 0xc009289018, 0xc00672d7a0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/bool_vec_to_sel.go:149 +0x55
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BoolVecToSelOp).Next(0xc01910dec0, 0x5a6dfa0, 0xc009289018, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/bool_vec_to_sel.go:58 +0x82
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*selectInOpBytes).Next(0xc01910df80, 0x5a6dfa0, 0xc009289018, 0x2cae034, 0x46b3fc0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/select_in.eg.go:639 +0x1f0
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*selectInOpBytes).Next(0xc01910dfc0, 0x5a6dfa0, 0xc009289018, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/select_in.eg.go:639 +0x1f0
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selNEBytesBytesConstOp).Next(0xc021098c80, 0x5a6dfa0, 0xc009289018, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:10156 +0x1e8
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc001f184b0, 0x5a6dfa0, 0xc009289018, 0xc02a3c8400, 0xa9fbc6)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc001f185a0, 0x5a6dfa0, 0xc009289018, 0xc0210985b8, 0x7f8b0f0869b8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).Next(0xc0204a7380, 0x5a6dfa0, 0xc009289018, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/stats.go:86 +0x5d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc01104b800, 0xc778af425b460, 0x3b197b7, 0x3b197b72a3c8590)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:290 +0x163
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:315
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc02a3c8558, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc01104b800, 0xed85a9554, 0x0, 0xc02a3c85c0, 0x462518)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:321 +0x5b
github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*inputStatCollector).Next(0xc00a6a3300, 0x8a28a28a28a2, 0x36, 0x8, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/rowexec/stats.go:67 +0x59
github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput(0xc012260000, 0x10, 0x10, 0x1, 0x5b8a680, 0xc02a3c8738)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:562 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next(0xc012260000, 0x1, 0x10, 0xc00aafef00, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:517 +0x1bb
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc010481200, 0x5a6dfa0, 0xc009288cd8, 0x5a90100, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:199 +0x10b
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).Next(0xc0204a7bc0, 0x5a6dfa0, 0xc009288cd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/stats.go:86 +0x5d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc00a6a3440, 0x5a6dfa0, 0xc009288cd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/is_null_ops.eg.go:241 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc006e88500, 0x5a6dfa0, 0xc009288cd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.constBoolOp.Next(0x5a69060, 0xc006e88500, 0xc00672c930, 0x10, 0x0, 0x5a6dfa0, 0xc009288cd8, 0xc00bea0110, 0xc00672d7d0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/const.eg.go:164 +0x63
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc006e88540, 0x5a6dfa0, 0xc009288cd8, 0x0, 0xc02a3c8b78)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*defaultBuiltinFuncOperator).Next(0xc00caec990, 0x5a6dfa0, 0xc009288cd8, 0xc02a3c8c58, 0xa9fbc6)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/builtin_funcs.go:48 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc001f18960, 0x5a6dfa0, 0xc009288cd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).Next(0xc0204a7c80, 0x5a6dfa0, 0xc009288cd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/stats.go:86 +0x5d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*allSpooler).spool(0xc01eb59b00, 0x5a6dfa0, 0xc009288cd8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:140 +0x5f
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*sortOp).Next(0xc00e8415c0, 0x5a6dfa0, 0xc009288cd8, 0x0, 0xc02a3c8ea8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:261 +0xbf
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc02a3c8f28, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc006d02e80, 0x5a6dfa0, 0xc009288cd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).Next(0xc00e841800, 0x5a6dfa0, 0xc009288cd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/stats.go:86 +0x5d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*unorderedDistinct).Next(0xc00672ced0, 0x5a6dfa0, 0xc009288cd8, 0x5, 0x10)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/unordered_distinct.go:72 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc02a3c90b8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc006d03100, 0x5a6dfa0, 0xc009288cd8, 0x2a36e7e, 0xc01910de80)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc006e89400, 0x5a6dfa0, 0xc009288cd8, 0xc00672c600, 0x8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*defaultBuiltinFuncOperator).Next(0xc00caecb40, 0x5a6dfa0, 0xc009288cd8, 0xc02a3c9268, 0xa9fbc6)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/builtin_funcs.go:48 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc02038c0f0, 0x5a6dfa0, 0xc009288cd8, 0x47dea5, 0x87471f8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).Next(0xc00e841a40, 0x5a6dfa0, 0xc009288cd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/stats.go:86 +0x5d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).spool(0xc00bfc4a00, 0x5a6dfa0, 0xc009288cd8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sorttopk.go:147 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).Next(0xc00bfc4a00, 0x5a6dfa0, 0xc009288cd8, 0x0, 0x2000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sorttopk.go:118 +0xee
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc02a3c9500, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc006d03180, 0x5a6dfa0, 0xc009288cd8, 0xc778af4256d25, 0x3b1507b)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc02038c1e0, 0x5a6dfa0, 0xc009288cd8, 0x0, 0x2d0506a)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*limitOp).Next(0xc00672d5c0, 0x5a6dfa0, 0xc009288cd8, 0xc01025e600, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/limit.go:53 +0x55
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).Next(0xc00e841bc0, 0x5a6dfa0, 0xc009288cd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/stats.go:86 +0x5d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc00f435200, 0xc00672d5c0, 0xc02a3c96d8, 0x2d229ef)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:290 +0x163
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:315
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc02a3c9730, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc00f435200, 0x5a6dfa0, 0xc009288cd8, 0x2d229c0, 0x7f8b0dd7deb0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:321 +0x5b
github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run(0x5a6dfa0, 0xc009288cd8, 0x5a745e0, 0xc00f435200, 0x5a16ca0, 0xc016d2b180)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:171 +0x35
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc00f435200, 0x5a4e420, 0xc01910db00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:781 +0x96
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216995439 [chan receive, 32 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0017dcd00, 0x5a4e4e0, 0xc011bee990, 0xc0126bf160, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc011bee990, 0x5ab2120, 0xc0102b5500, 0xc00c636009, 0xd, 0xc011bee960, 0x59fb820, 0xc011bee3f0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc011bee990, 0x5ab21e0, 0xc00df96f60, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc00df96f60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc00df96f60, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 216090970 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc021a62510, 0x1688bcc286d61d9a, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc021a62510, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc021a624e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc021a624e0, 0x4c245c5, 0x10, 0xc021b0f6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc021a624e0, 0xc021570aa0, 0xc021a624e0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc021a62480, 0xc021570aa0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc021a62480, 0x4bc7440, 0xc021570aa0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc021a62480, 0x4bc7440, 0xc021570aa0, 0xc022671860, 0xc022671880, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc021a62480, 0x4bc7440, 0xc021570aa0, 0xc021b0f990, 0xca72c8, 0x48dd840, 0xc01ac62280)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc021a62480, 0x4bc7440, 0xc021570aa0, 0xc022671860, 0xc01ac62280, 0xc01ac62280, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc021a62480, 0x4bc7440, 0xc021570aa0, 0x0, 0xc00758a000, 0x0, 0xc015b74420)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc021a62480, 0x4bc7440, 0xc021570aa0, 0xc022671860, 0xc01ac62240, 0xc021b0fac8, 0xca72c8, 0x48dd840, 0xc01ac62240)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc021a62480, 0x4bc7440, 0xc021570aa0, 0xc022671860, 0xc022671880, 0xc003253ba0, 0x661046, 0x491f620, 0xc021a62480)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc021a62480, 0xc015b743c0, 0xc001681820, 0x5a4e4e0, 0xc021a62480, 0xc0215b3800, 0x32e)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02087aa00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02087aa00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02087aa00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218095727 [select]:
github.com/cockroachdb/cockroach/pkg/cli.runStart.func6(0xc000c04b80, 0xc000402de0)
/go/src/github.com/cockroachdb/cockroach/pkg/cli/start.go:829 +0x1e6
created by github.com/cockroachdb/cockroach/pkg/cli.runStart
/go/src/github.com/cockroachdb/cockroach/pkg/cli/start.go:825 +0xd2e
goroutine 216085778 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).GetRangeInfo(0xc019f59200, 0x5a4e4e0, 0xc01f764540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica.go:881 +0x4a8
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).maybeAddRangeInfoToResponse(0xc019f59200, 0x5a4e4e0, 0xc01f764540, 0xc001f914a0, 0xc0163e9680)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:227 +0x67
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc019f59200, 0x5a4e4e0, 0xc01f764510, 0x5c79, 0xc001f914a0, 0x0, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:123 +0x3aa
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:34
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01f7644e0, 0x1688bcbdf4b7f568, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:195 +0x5cc
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01f7644e0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01f7644e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01f7644e0, 0x4c245c5, 0x10, 0xc01d5a0880, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01f7644e0, 0xc001f912c0, 0xc01f7644e0, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01f7644b0, 0xc001f912c0, 0xc0142daf40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00dd25d80, 0x5a4e4e0, 0xc01f7644b0, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00dd25d80, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1145 +0x3e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:784 +0x5e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0149f88d0, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc0149f8898, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc0149f8868, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:126 +0x8db
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc0149f87c8, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc0149f87c8, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc0149f86f8, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc0149f86d8, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0149f8640, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc0149f8480, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00a00ce10, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcher.func1(0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:217 +0x85
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc012d51260, 0x5a4e420, 0xc0142daf40, 0x0, 0x6bebe)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:379 +0x739
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc012d51260, 0x5a4e420, 0xc0142daf40, 0x7, 0x8, 0x0, 0x0, 0x39, 0x6bf62, 0xffffffffffffffff, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:480 +0x1ab
github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc00a00d560, 0x5a4e420, 0xc0142daf40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:129 +0x104
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).nextBatch(0xc00a0ba1e0, 0x5a4e420, 0xc0142daf40, 0xc024d6aac0, 0x11044, 0xe9ddc, 0x27)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:803 +0xb9
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:785
github.com/cockroachdb/cockroach/pkg/sql/colmem.(*Allocator).PerformOperation(0xc0274e9140, 0xc015994500, 0x28, 0x28, 0xc01d5a5bb0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colmem/allocator.go:300 +0x59
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc00a0ba1e0, 0x5a4e420, 0xc0142daf40, 0x0, 0xc0142daf00, 0x0, 0xc01d5a5c38)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:777 +0xa5
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc0163e8c80, 0x5a4e420, 0xc0142daf40, 0xc0274e96b0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x87
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc0274e9200, 0x5a4e420, 0xc0142daf40, 0xc0274e96b0, 0xc01d5a5d00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:48 +0x69
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc00fcf5e50, 0x5a4e420, 0xc0142daf40, 0x20, 0x5a69060)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selEQInt64Int64ConstOp).Next(0xc00a00cf30, 0x5a4e420, 0xc0142daf40, 0x5b8a880, 0xc01d5a5df0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:6114 +0xf7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selGEInt64Int64ConstOp).Next(0xc00a00cfc0, 0x5a4e420, 0xc0142daf40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:54994 +0xf7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selGTTimestampTimestampConstOp).Next(0xc01f106be0, 0x5a4e420, 0xc0142daf40, 0xc01d5a5f80, 0x4616aa)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:48146 +0x237
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc014f6b7e0, 0x5a4e420, 0xc0142daf40, 0xc016136000, 0x7f8b0d11ee48)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/is_null_ops.eg.go:241 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BatchSchemaSubsetEnforcer).Next(0xc01aa48120, 0x5a4e420, 0xc0142daf40, 0xc00009c800, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:212 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*bufferOp).advance(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/buffer.go:62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*caseOp).Next(0xc00a00d170, 0x5a4e420, 0xc0142daf40, 0x400, 0xc024d6ac00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/case.go:109 +0x6b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.selBoolOp.Next(0x5a518a0, 0xc00a00d170, 0x0, 0x0, 0xc0142db7c0, 0xf, 0x5a4e420, 0xc0142daf40, 0x7fffffffffffffff, 0xc024d6ac00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/bool_vec_to_sel.go:149 +0x55
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BoolVecToSelOp).Next(0xc0142db7c0, 0x5a4e420, 0xc0142daf40, 0xc020aed780, 0xc0165dc800)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/bool_vec_to_sel.go:58 +0x82
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*selectInOpBytes).Next(0xc0142db800, 0x5a4e420, 0xc0142daf40, 0xc024d6a180, 0xc00dadb800)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/select_in.eg.go:639 +0x1f0
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*selectInOpBytes).Next(0xc0142db880, 0x5a4e420, 0xc0142daf40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/select_in.eg.go:639 +0x1f0
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selNEBytesBytesConstOp).Next(0xc01f107220, 0x5a4e420, 0xc0142daf40, 0x1c6f445, 0xc01d5a6710)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:10156 +0x1e8
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc00fcf5ea0, 0x5a4e420, 0xc0142daf40, 0xbd, 0x7f8b0dca8690)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc00fcf5f40, 0x5a4e420, 0xc0142daf40, 0x19d8503, 0xc00ac9e180)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc019323800, 0x19dad57, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:290 +0x163
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:315
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01d5a6758, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc019323800, 0xc0142cf3e0, 0x1, 0xc0274e9860, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:321 +0x5b
github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput(0xc015f49300, 0x10, 0x10, 0x1, 0x5b8a680, 0xc01d5a68c8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:562 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next(0xc015f49300, 0x1, 0x10, 0xc01058c300, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:517 +0x1bb
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc013382900, 0x5a4e420, 0xc0142daf40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:199 +0x10b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc01fc751a0, 0x5a4e420, 0xc0142daf40, 0xc002f60ea0, 0xc0025754a0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/is_null_ops.eg.go:241 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc0142db8c0, 0x5a4e420, 0xc0142daf40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.constBoolOp.Next(0x5a69060, 0xc0142db8c0, 0xc0274e9980, 0x10, 0x0, 0x5a4e420, 0xc0142daf40, 0xc00a00d560, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/const.eg.go:164 +0x63
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc0142db900, 0x5a4e420, 0xc0142daf40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*defaultBuiltinFuncOperator).Next(0xc00a00d320, 0x5a4e420, 0xc0142daf40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/builtin_funcs.go:48 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc0117ce9b0, 0x5a4e420, 0xc0142daf40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*allSpooler).spool(0xc01aa48660, 0x5a4e420, 0xc0142daf40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:140 +0x5f
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*sortOp).Next(0xc00c4d9ec0, 0x5a4e420, 0xc0142daf40, 0x7c8fe01, 0xc0163e8cc0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:261 +0xbf
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01d5a7018, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc0163e9300, 0x5a4e420, 0xc0142daf40, 0xc01aa48120, 0xc01d5a7088)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*unorderedDistinct).Next(0xc0274e9f50, 0x5a4e420, 0xc0142daf40, 0x5a69060, 0xc0142db100)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/unordered_distinct.go:72 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01d5a7158, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc0163e9400, 0x5a4e420, 0xc0142daf40, 0x2a09593, 0xc0274e9320)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc020aec480, 0x5a4e420, 0xc0142daf40, 0x0, 0xc01d5a7228)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*defaultBuiltinFuncOperator).Next(0xc00a00d4d0, 0x5a4e420, 0xc0142daf40, 0x203007, 0x203007)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/builtin_funcs.go:48 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc0117ceaf0, 0x5a4e420, 0xc0142daf40, 0x20300200000000, 0x7f8afe7eefff)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).spool(0xc013942e00, 0x5a4e420, 0xc0142daf40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sorttopk.go:147 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).Next(0xc013942e00, 0x5a4e420, 0xc0142daf40, 0xc01d5a74d0, 0x4616aa)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sorttopk.go:118 +0xee
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01d5a7550, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc0163e9480, 0x5a4e420, 0xc0142daf40, 0xc01d5a7610, 0x2906f85)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc0117ceb90, 0x5a4e420, 0xc0142daf40, 0xc01aa492c0, 0x6)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*limitOp).Next(0xc002574f00, 0x5a4e420, 0xc0142daf40, 0xc01d5a76a8, 0x2a238d3)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/limit.go:53 +0x55
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc00f435800, 0xc0117ceb90, 0xc01d5a76d8, 0x2d229ef)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:290 +0x163
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:315
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01d5a7730, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc00f435800, 0x5a4e420, 0xc0142daf40, 0x2d229c0, 0x5a51b20)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:321 +0x5b
github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run(0x5a4e420, 0xc0142daf40, 0x5a745e0, 0xc00f435800, 0x5a16ca0, 0xc006f1e380)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:171 +0x35
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc00f435800, 0x5a4e420, 0xc0142daf40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:781 +0x96
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc0146af9e0, 0x5a4e420, 0xc0142daf40, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc012d50c40, 0xc00a00ce10, 0xc0163e8780, 0xc006f1e380, 0xc013499ff8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc01b11f050, 0xc013499ff8, 0xc012d50c40, 0xc00a00ce10, 0x5a507e0, 0xc01f1068c0, 0x0, 0xc006f1e380, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc013499900, 0x5a4e4e0, 0xc01b11f050, 0xc013499de0, 0x3, 0x7f8b0dd7dcb0, 0xc005e27880, 0xc01d5a8001, 0xc01023c898, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc013499900, 0x5a4e4e0, 0xc01b11f050, 0xc013499de0, 0x7f8b0dd7dcb0, 0xc005e27880, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc013499900, 0x5a4e4e0, 0xc01b11f050, 0x5a6ce20, 0xc00f5eba90, 0xc005ca61a0, 0x269, 0xb, 0x1, 0xc00a3cd860, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 217601898 [semacquire, 50 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01b1dbd40, 0x1688c4896cf3bfc0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01b1dbd40, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01b1dbd10, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01b1dbd10, 0x4c245c5, 0x10, 0xc0046d16c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01b1dbd10, 0xc0255dc780, 0xc01b1dbd10, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01b1dbcb0, 0xc0255dc780, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01b1dbcb0, 0x4bc7440, 0xc0255dc780, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01b1dbcb0, 0x4bc7440, 0xc0255dc780, 0xc01feebba0, 0xc01feebbc0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01b1dbcb0, 0x4bc7440, 0xc0255dc780, 0xc0046d1990, 0xca72c8, 0x48dd840, 0xc0220692c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01b1dbcb0, 0x4bc7440, 0xc0255dc780, 0xc01feebba0, 0xc0220692c0, 0xc0220692c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01b1dbcb0, 0x4bc7440, 0xc0255dc780, 0x0, 0xc00fba5800, 0x0, 0xc016913800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01b1dbcb0, 0x4bc7440, 0xc0255dc780, 0xc01feebba0, 0xc022069280, 0xc0046d1ac8, 0xca72c8, 0x48dd840, 0xc022069280)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01b1dbcb0, 0x4bc7440, 0xc0255dc780, 0xc01feebba0, 0xc01feebbc0, 0xc0020f8ba0, 0x661046, 0x491f620, 0xc01b1dbcb0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01b1dbcb0, 0xc0169137a0, 0xc001681820, 0x5a4e4e0, 0xc01b1dbcb0, 0xc00a67f800, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00a31b700, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00a31b700, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00a31b700)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216093757 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc002a68180, 0x1688bcc40cda9909, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc002a68180, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc002a68090, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc002a68090, 0x4c245c5, 0x10, 0xc0039236c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc002a68090, 0xc01e8a85a0, 0xc002a68090, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc004b517d0, 0xc01e8a85a0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc004b517d0, 0x4bc7440, 0xc01e8a85a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc004b517d0, 0x4bc7440, 0xc01e8a85a0, 0xc022080320, 0xc022080340, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc004b517d0, 0x4bc7440, 0xc01e8a85a0, 0xc003923990, 0xca72c8, 0x48dd840, 0xc01700c2c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc004b517d0, 0x4bc7440, 0xc01e8a85a0, 0xc022080320, 0xc01700c2c0, 0xc01700c2c0, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc004b517d0, 0x4bc7440, 0xc01e8a85a0, 0x0, 0x87067e0, 0x0, 0xc01af709c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc004b517d0, 0x4bc7440, 0xc01e8a85a0, 0xc022080320, 0xc01700c280, 0xc003923ac8, 0xca72c8, 0x48dd840, 0xc01700c280)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc004b517d0, 0x4bc7440, 0xc01e8a85a0, 0xc022080320, 0xc022080340, 0xc00252eba0, 0x661046, 0x491f620, 0xc004b517d0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc004b517d0, 0xc01af70960, 0xc001681820, 0x5a4e4e0, 0xc004b517d0, 0xc0118b5800, 0x357)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc015d67100, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc015d67100, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc015d67100)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216508688 [semacquire, 157 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01b410c60, 0x1688beaccbe220b4, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01b410c60, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01b410c00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01b410c00, 0x4c245c5, 0x10, 0xc004a176c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01b410c00, 0xc002bcfb80, 0xc01b410c00, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01b4108d0, 0xc002bcfb80, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01b4108d0, 0x4bc7440, 0xc002bcfb80, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01b4108d0, 0x4bc7440, 0xc002bcfb80, 0xc01fa536c0, 0xc01fa536e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01b4108d0, 0x4bc7440, 0xc002bcfb80, 0xc004a17990, 0xca72c8, 0x48dd840, 0xc0100fcbc0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01b4108d0, 0x4bc7440, 0xc002bcfb80, 0xc01fa536c0, 0xc0100fcbc0, 0xc0100fcbc0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01b4108d0, 0x4bc7440, 0xc002bcfb80, 0x0, 0xc000100400, 0x0, 0xc0165d2360)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01b4108d0, 0x4bc7440, 0xc002bcfb80, 0xc01fa536c0, 0xc0100fcb80, 0xc004a17ac8, 0xca72c8, 0x48dd840, 0xc0100fcb80)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01b4108d0, 0x4bc7440, 0xc002bcfb80, 0xc01fa536c0, 0xc01fa536e0, 0xc002f74ba0, 0x661046, 0x491f620, 0xc01b4108d0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01b4108d0, 0xc0165d2240, 0xc001681820, 0x5a4e4e0, 0xc01b4108d0, 0xc00dfea000, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f559f00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f559f00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f559f00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216798959 [semacquire, 132 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02dca6180, 0x1688c008358e55a5, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02dca6180, 0x1688c008358e55a5, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02dca6150, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02dca6150, 0x4c245c5, 0x10, 0xc0055c16c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02dca6150, 0xc002a0c0a0, 0xc02dca6150, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02dca60f0, 0xc002a0c0a0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02dca60f0, 0x4bc7440, 0xc002a0c0a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02dca60f0, 0x4bc7440, 0xc002a0c0a0, 0xc00ed1d740, 0xc00ed1d760, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02dca60f0, 0x4bc7440, 0xc002a0c0a0, 0xc0055c1990, 0xca72c8, 0x48dd840, 0xc022ccb040)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02dca60f0, 0x4bc7440, 0xc002a0c0a0, 0xc00ed1d740, 0xc022ccb040, 0xc022ccb040, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02dca60f0, 0x4bc7440, 0xc002a0c0a0, 0x0, 0x87067e0, 0x0, 0xc0143acf00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02dca60f0, 0x4bc7440, 0xc002a0c0a0, 0xc00ed1d740, 0xc022ccb000, 0xc0055c1ac8, 0xca72c8, 0x48dd840, 0xc022ccb000)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02dca60f0, 0x4bc7440, 0xc002a0c0a0, 0xc00ed1d740, 0xc00ed1d760, 0xc0037e3ba0, 0x661046, 0x491f620, 0xc02dca60f0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02dca60f0, 0xc0143acea0, 0xc001681820, 0x5a4e4e0, 0xc02dca60f0, 0xc008422000, 0x431)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02097db00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02097db00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02097db00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218080033 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02b048750, 0x1688c728017e5550, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02b048750, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02b048720, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02b048720, 0x4c245c5, 0x10, 0xc01f84d6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02b048720, 0xc028056fa0, 0xc02b048720, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02b048690, 0xc028056fa0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02b048690, 0x4bc7440, 0xc028056fa0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02b048690, 0x4bc7440, 0xc028056fa0, 0xc0155fcbe0, 0xc0155fcc40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02b048690, 0x4bc7440, 0xc028056fa0, 0xc01f84d990, 0xca72c8, 0x48dd840, 0xc01b140880)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02b048690, 0x4bc7440, 0xc028056fa0, 0xc0155fcbe0, 0xc01b140880, 0xc01b140880, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02b048690, 0x4bc7440, 0xc028056fa0, 0x0, 0xc00e8e4800, 0x0, 0xc0280ef080)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02b048690, 0x4bc7440, 0xc028056fa0, 0xc0155fcbe0, 0xc01b140840, 0xc01f84dac8, 0xca72c8, 0x48dd840, 0xc01b140840)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02b048690, 0x4bc7440, 0xc028056fa0, 0xc0155fcbe0, 0xc0155fcc40, 0xc0013f1ba0, 0x661046, 0x491f620, 0xc02b048690)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02b048690, 0xc0280ef020, 0xc001681820, 0x5a4e4e0, 0xc02b048690, 0xc01de19800, 0x317)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc027a74a00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc027a74a00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc027a74a00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216997568 [chan receive, 31 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc012749b00, 0x5a4e4e0, 0xc0136e4570, 0xc0072f7af0, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc0136e4570, 0x5ab2120, 0xc00f446a80, 0xc014396028, 0xd, 0xc0136e4540, 0x59fb820, 0xc0110dbec0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc0136e4570, 0x5ab21e0, 0xc016c1b4a0, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc016c1b4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc016c1b4a0, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 217540020 [semacquire, 55 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc015410a80, 0x1688c43eeb101bcd, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc015410a80, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0154109c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0154109c0, 0x4c245c5, 0x10, 0xc0045596c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0154109c0, 0xc01e846dc0, 0xc0154109c0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc015410870, 0xc01e846dc0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc015410870, 0x4bc7440, 0xc01e846dc0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc015410870, 0x4bc7440, 0xc01e846dc0, 0xc01aaf0940, 0xc01aaf0960, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc015410870, 0x4bc7440, 0xc01e846dc0, 0xc004559990, 0xca72c8, 0x48dd840, 0xc00f34e0c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc015410870, 0x4bc7440, 0xc01e846dc0, 0xc01aaf0940, 0xc00f34e0c0, 0xc00f34e0c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc015410870, 0x4bc7440, 0xc01e846dc0, 0x0, 0xc003d76000, 0x0, 0xc016ee1380)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc015410870, 0x4bc7440, 0xc01e846dc0, 0xc01aaf0940, 0xc00f34e080, 0xc004559ac8, 0xca72c8, 0x48dd840, 0xc00f34e080)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc015410870, 0x4bc7440, 0xc01e846dc0, 0xc01aaf0940, 0xc01aaf0960, 0xc0020f8ba0, 0x661046, 0x491f620, 0xc015410870)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc015410870, 0xc016ee1320, 0xc001681820, 0x5a4e4e0, 0xc015410870, 0xc013e2b800, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc010ca9300, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc010ca9300, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc010ca9300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218080158 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc00b3c3890, 0x1688c728094e26f0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc00b3c3890, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00b3c3860, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00b3c3860, 0x4c245c5, 0x10, 0xc01fe396c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00b3c3860, 0xc020bd39a0, 0xc00b3c3860, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00b3c3710, 0xc020bd39a0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc00b3c3710, 0x4bc7440, 0xc020bd39a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc00b3c3710, 0x4bc7440, 0xc020bd39a0, 0xc016e46360, 0xc016e46380, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00b3c3710, 0x4bc7440, 0xc020bd39a0, 0xc01fe39990, 0xca72c8, 0x48dd840, 0xc011c24f80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc00b3c3710, 0x4bc7440, 0xc020bd39a0, 0xc016e46360, 0xc011c24f80, 0xc011c24f80, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00b3c3710, 0x4bc7440, 0xc020bd39a0, 0x0, 0xc00758a000, 0x0, 0xc009a8a600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc00b3c3710, 0x4bc7440, 0xc020bd39a0, 0xc016e46360, 0xc011c24f40, 0xc01fe39ac8, 0xca72c8, 0x48dd840, 0xc011c24f40)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc00b3c3710, 0x4bc7440, 0xc020bd39a0, 0xc016e46360, 0xc016e46380, 0xc00301dba0, 0x661046, 0x491f620, 0xc00b3c3710)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc00b3c3710, 0xc009a8a5a0, 0xc001681820, 0x5a4e4e0, 0xc00b3c3710, 0xc01f59e600, 0x304)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0298f9500, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0298f9500, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc0298f9500)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216089522 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc00b418f30, 0x1688bcc201004b94, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc00b418f30, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00b418f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00b418f00, 0x4c245c5, 0x10, 0xc010c236c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00b418f00, 0xc00e9695e0, 0xc00b418f00, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00b418e70, 0xc00e9695e0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc00b418e70, 0x4bc7440, 0xc00e9695e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc00b418e70, 0x4bc7440, 0xc00e9695e0, 0xc019f9a8c0, 0xc019f9a8e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00b418e70, 0x4bc7440, 0xc00e9695e0, 0xc010c23990, 0xca72c8, 0x48dd840, 0xc00ce40600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc00b418e70, 0x4bc7440, 0xc00e9695e0, 0xc019f9a8c0, 0xc00ce40600, 0xc00ce40600, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00b418e70, 0x4bc7440, 0xc00e9695e0, 0x0, 0xc000100800, 0x0, 0xc00ea5b800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc00b418e70, 0x4bc7440, 0xc00e9695e0, 0xc019f9a8c0, 0xc00ce405c0, 0xc010c23ac8, 0xca72c8, 0x48dd840, 0xc00ce405c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc00b418e70, 0x4bc7440, 0xc00e9695e0, 0xc019f9a8c0, 0xc019f9a8e0, 0xc000f91ba0, 0x661046, 0x491f620, 0xc00b418e70)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc00b418e70, 0xc00ea5b7a0, 0xc001681820, 0x5a4e4e0, 0xc00b418e70, 0xc010505200, 0x336)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00e944b00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00e944b00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00e944b00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217929605 [semacquire, 17 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01980fbf0, 0x1688c652f2384b00, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01980fbf0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01980fbc0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01980fbc0, 0x4c245c5, 0x10, 0xc00cefb6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01980fbc0, 0xc014a70dc0, 0xc01980fbc0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01980fb60, 0xc014a70dc0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01980fb60, 0x4bc7440, 0xc014a70dc0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01980fb60, 0x4bc7440, 0xc014a70dc0, 0xc00fd0cd40, 0xc00fd0cd60, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01980fb60, 0x4bc7440, 0xc014a70dc0, 0xc00cefb990, 0xca72c8, 0x48dd840, 0xc00b0d0680)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01980fb60, 0x4bc7440, 0xc014a70dc0, 0xc00fd0cd40, 0xc00b0d0680, 0xc00b0d0680, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01980fb60, 0x4bc7440, 0xc014a70dc0, 0x0, 0x87067e0, 0x0, 0xc01012d140)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01980fb60, 0x4bc7440, 0xc014a70dc0, 0xc00fd0cd40, 0xc00b0d0580, 0xc00cefbac8, 0xca72c8, 0x48dd840, 0xc00b0d0580)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01980fb60, 0x4bc7440, 0xc014a70dc0, 0xc00fd0cd40, 0xc00fd0cd60, 0xc0000a7ba0, 0x661046, 0x491f620, 0xc01980fb60)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01980fb60, 0xc01012d0e0, 0xc001681820, 0x5a4e4e0, 0xc01980fb60, 0xc00abc8c00, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc010ca8b00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc010ca8b00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc010ca8b00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216404088 [semacquire, 166 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0069f7c20, 0x1688be2de6d1c5c6, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0069f7c20, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0069f7bf0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0069f7bf0, 0x4c245c5, 0x10, 0xc004c316c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0069f7bf0, 0xc002704f00, 0xc0069f7bf0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0069f7b90, 0xc002704f00, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0069f7b90, 0x4bc7440, 0xc002704f00, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0069f7b90, 0x4bc7440, 0xc002704f00, 0xc024337a00, 0xc024337a20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0069f7b90, 0x4bc7440, 0xc002704f00, 0xc004c31990, 0xca72c8, 0x48dd840, 0xc0113a9700)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0069f7b90, 0x4bc7440, 0xc002704f00, 0xc024337a00, 0xc0113a9700, 0xc0113a9700, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0069f7b90, 0x4bc7440, 0xc002704f00, 0x0, 0x87067e0, 0x0, 0xc0017bff80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0069f7b90, 0x4bc7440, 0xc002704f00, 0xc024337a00, 0xc0113a96c0, 0xc004c31ac8, 0xca72c8, 0x48dd840, 0xc0113a96c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0069f7b90, 0x4bc7440, 0xc002704f00, 0xc024337a00, 0xc024337a20, 0xc007306ba0, 0x661046, 0x491f620, 0xc0069f7b90)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0069f7b90, 0xc0017bff20, 0xc001681820, 0x5a4e4e0, 0xc0069f7b90, 0xc003727200, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0142b2e00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0142b2e00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0142b2e00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217684450 [semacquire, 40 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc021a62ba0, 0x1688c50aa62f24ee, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc021a62ba0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc021a62b40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc021a62b40, 0x4c245c5, 0x10, 0xc024b976c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc021a62b40, 0xc01d4a6fa0, 0xc021a62b40, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc021a62ae0, 0xc01d4a6fa0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc021a62ae0, 0x4bc7440, 0xc01d4a6fa0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc021a62ae0, 0x4bc7440, 0xc01d4a6fa0, 0xc007aafda0, 0xc007aafdc0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc021a62ae0, 0x4bc7440, 0xc01d4a6fa0, 0xc024b97990, 0xca72c8, 0x48dd840, 0xc01388cac0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc021a62ae0, 0x4bc7440, 0xc01d4a6fa0, 0xc007aafda0, 0xc01388cac0, 0xc01388cac0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc021a62ae0, 0x4bc7440, 0xc01d4a6fa0, 0x0, 0xc024c79800, 0x0, 0xc012b4cde0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc021a62ae0, 0x4bc7440, 0xc01d4a6fa0, 0xc007aafda0, 0xc01388ca80, 0xc024b97ac8, 0xca72c8, 0x48dd840, 0xc01388ca80)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc021a62ae0, 0x4bc7440, 0xc01d4a6fa0, 0xc007aafda0, 0xc007aafdc0, 0xc001b07ba0, 0x661046, 0x491f620, 0xc021a62ae0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc021a62ae0, 0xc012b4cd80, 0xc001681820, 0x5a4e4e0, 0xc021a62ae0, 0xc0039ca600, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0150e4700, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0150e4700, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0150e4700)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216997685 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01530a3f0, 0x1688c18c680cefdb, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01530a3f0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01530a390, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01530a390, 0x4c245c5, 0x10, 0xc0056836c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01530a390, 0xc00e6adc20, 0xc01530a390, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01530a330, 0xc00e6adc20, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01530a330, 0x4bc7440, 0xc00e6adc20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01530a330, 0x4bc7440, 0xc00e6adc20, 0xc0073253c0, 0xc0073253e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01530a330, 0x4bc7440, 0xc00e6adc20, 0xc005683990, 0xca72c8, 0x48dd840, 0xc0127e1780)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01530a330, 0x4bc7440, 0xc00e6adc20, 0xc0073253c0, 0xc0127e1780, 0xc0127e1780, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01530a330, 0x4bc7440, 0xc00e6adc20, 0x0, 0x87067e0, 0x0, 0xc010bde900)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01530a330, 0x4bc7440, 0xc00e6adc20, 0xc0073253c0, 0xc0127e1640, 0xc005683ac8, 0xca72c8, 0x48dd840, 0xc0127e1640)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01530a330, 0x4bc7440, 0xc00e6adc20, 0xc0073253c0, 0xc0073253e0, 0xc02eebaba0, 0x661046, 0x491f620, 0xc01530a330)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01530a330, 0xc010bde8a0, 0xc001681820, 0x5a4e4e0, 0xc01530a330, 0xc011a58c00, 0x31d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc020242d00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc020242d00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc020242d00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216090478 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc020392870, 0x1688bcc273e542f1, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc020392870, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc020392870, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc020392870, 0x4c245c5, 0x10, 0xc022fce410, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc020392870, 0xc020002f00, 0xc020392870, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc020392840, 0xc020002f00, 0xc01637d000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc010be1f00, 0x5a4e4e0, 0xc020392840, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc010be1f00, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc01fd0b650, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc01fd0b618, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc01fd0b5e8, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc01fd0b548, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc01fd0b548, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc01fd0b478, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc01fd0b458, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc01fd0b3c0, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc01fd0b200, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc01e8df050, 0x5a4e420, 0xc01637d000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc01637d000, 0xc022fd3208, 0xc012560500, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc01e8df050, 0x5a4e420, 0xc01637d000, 0xc012560500, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc01e8df050, 0x5a4e420, 0xc01637d000, 0xc012560500, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc00e74a028, 0x5a4e420, 0xc01637d000, 0xc0164de600, 0xc01ea784e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00e74a000, 0x5a4e420, 0xc01637d000, 0xc0164de600, 0xc01ea784e0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc00aa2ab20, 0x5a4e420, 0xc01637d000, 0xc0164de600, 0xc01ea784e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc00aa2ab20, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc022fd35a0, 0x4bfc8f0, 0x5, 0x5a50860, 0xc00aa2ab20)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc022fd35a0, 0x5a50860, 0xc00aa2ab20, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc022fd35a0, 0x5a50860, 0xc00aa2ab20, 0x0, 0xc01e970140)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc01637d000, 0xc0164de600, 0xc01ea784e0, 0x5a50860, 0xc00aa2ab20, 0x5a4e420, 0xc01637d000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc02004a000, 0x5a4e420, 0xc01637d000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc01fd12400)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc022fd3768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc02004ac00, 0x5a4e420, 0xc01637d000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc02004ac00, 0x5a4e420, 0xc01637d000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc020018ea0, 0x5a4e420, 0xc01637d000, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc004af09a0, 0xc01e8df050, 0xc01ebd7800, 0xc00fe50000, 0xc01ea786f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc01fd5aed0, 0xc01ea786f8, 0xc004af09a0, 0xc01e8df050, 0x5a50860, 0xc00aa2ab20, 0x0, 0xc00fe50000, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc01ea78000, 0x5a4e4e0, 0xc01fd5aed0, 0xc01ea784e0, 0x2, 0x7f8b0dd7dcb0, 0xc016e57c80, 0xc022fd4000, 0xc019e2a058, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc01ea78000, 0x5a4e4e0, 0xc01fd5aed0, 0xc01ea784e0, 0x7f8b0dd7dcb0, 0xc016e57c80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc01ea78000, 0x5a4e4e0, 0xc01fd5aed0, 0x5a6dda0, 0xc01fcca900, 0xc01ee54719, 0x1e9, 0x11, 0x1, 0xc02b08e000, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc01ea78000, 0x5a4e420, 0xc0224c9940, 0x5a6dda0, 0xc01fcca900, 0xc01ee54719, 0x1e9, 0x11, 0x1, 0xc02b08e000, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc01ea78000, 0x5a4e420, 0xc0224c9940, 0xc02b08e000, 0xc0200185a0, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc01ee54a8a, 0x0, 0x0, 0x141c427, 0xed85a80c9, 0x0, 0xc01ea78000, 0xc022fd5a20, 0xc022fd5a10, 0x6e, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc01ea78000, 0x5a4e420, 0xc0224c9940, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc01ea78000, 0x5a4e420, 0xc011193c80, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc00a925340, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc011193c80, 0xc01ea78000, 0x5400, 0x15000, 0xc001805a40, 0xc00a925340, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc00d307f01, 0xc008a664a0, 0x5a4e420, 0xc011193c80, 0xc00a925340, 0xc001a38000, 0xc016e57600, 0x5ab5540, 0xc01d4ced80, 0xc006846a80, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 218080758 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc0239467f8, 0x1688c7286874c8dc, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc0239467f8, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01a7ff7a0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01a7ff7a0, 0x4c245c5, 0x10, 0xc028a2e3f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01a7ff7a0, 0xc027e6c1e0, 0xc01a7ff7a0, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01a7ff770, 0xc027e6c1e0, 0xc0239464b8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc011cc1940, 0x5a4e4e0, 0xc01a7ff770, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc011cc1940, 0x5a6dfa0, 0xc0239464b8, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc0239464b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc0239464b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc0239464b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc0239464b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc023946318, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc02ac3e450, 0x5a6dfa0, 0xc023946318, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc02ac3e418, 0x5a6dfa0, 0xc023946318, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc02ac3e3e8, 0x5a6dfa0, 0xc023946318, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc02ac3e348, 0x5a6dfa0, 0xc023946318, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc02ac3e348, 0x5a6dfa0, 0xc023946318, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc02ac3e278, 0x5a6dfa0, 0xc023946318, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc02ac3e258, 0x5a6dfa0, 0xc023946318, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc02ac3e1c0, 0x5a6dfa0, 0xc023946318, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc02ac3e000, 0x5a4e420, 0xc010d2cac0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc010d2cac0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc023d42bd0, 0x5a4e420, 0xc010d2cac0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc010d2cac0, 0xc028a331f0, 0xc0150d6000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc023d42bd0, 0x5a4e420, 0xc010d2cac0, 0xc0150d6000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc023d42bd0, 0x5a4e420, 0xc010d2cac0, 0xc0150d6000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc023d72628, 0x5a4e420, 0xc010d2cac0, 0xc02204ef00, 0xc0223e30e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc023d72600, 0x5a4e420, 0xc010d2cac0, 0xc02204ef00, 0xc0223e30e0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc00a47dce0, 0x5a4e420, 0xc010d2cac0, 0xc02204ef00, 0xc0223e30e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc00a47dce0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc028a33588, 0x4bfc8f0, 0x5, 0x5a50860, 0xc00a47dce0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc028a33588, 0x5a50860, 0xc00a47dce0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc028a33588, 0x5a50860, 0xc00a47dce0, 0xc, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc010d2cac0, 0xc02204ef00, 0xc0223e30e0, 0x5a50860, 0xc00a47dce0, 0x5a4e420, 0xc010d2cac0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc023d72c00, 0x5a4e420, 0xc010d2cac0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc023ad7600)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc028a33768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc023d73800, 0x5a4e420, 0xc010d2cac0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc023d73800, 0x5a4e420, 0xc010d2cac0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc0193c6120, 0x5a4e420, 0xc010d2cac0, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc00d94c620, 0xc023d42bd0, 0xc024028380, 0xc0229cd500, 0xc0223e32f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc023a98e78, 0xc0223e32f8, 0xc00d94c620, 0xc023d42bd0, 0x5a50860, 0xc00a47dce0, 0x0, 0xc0229cd500, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0223e2c00, 0x5a6dfa0, 0xc023a98e78, 0xc0223e30e0, 0x2, 0x7f8b0dd7dcb0, 0xc023ad4f80, 0xc028a34000, 0xc025457978, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0223e2c00, 0x5a6dfa0, 0xc023a98e78, 0xc0223e30e0, 0x7f8b0dd7dcb0, 0xc023ad4f80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0223e2c00, 0x5a6dfa0, 0xc023a98e78, 0x5a6dda0, 0xc023a85c80, 0xc0235d9285, 0x1e9, 0x11, 0x1, 0xc023ade690, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0223e2c00, 0x5a4e420, 0xc015327840, 0x5a6dda0, 0xc023a85c80, 0xc0235d9285, 0x1e9, 0x11, 0x1, 0xc023ade690, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0223e2c00, 0x5a4e420, 0xc015327840, 0xc023ade690, 0xc0137ab680, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc0235d95f7, 0x0, 0x0, 0x1ea3151, 0xed85aad72, 0x0, 0xc0223e2c00, 0xc028a35a20, 0xc028a35a10, 0x27, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0223e2c00, 0x5a4e420, 0xc015327840, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0223e2c00, 0x5a4e420, 0xc0153260c0, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc007e47e90, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc0153260c0, 0xc0223e2c00, 0x5400, 0x15000, 0xc001805a40, 0xc007e47e90, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc014260156, 0xc008fc2f60, 0x5a4e420, 0xc0153260c0, 0xc007e47e90, 0xc001a38000, 0xc023ad4900, 0x5ab5540, 0xc023a84c80, 0xc025456900, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216605210 [semacquire, 149 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc00fab0540, 0x1688bf238ad6bdd5, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc00fab0540, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00fab0510, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00fab0510, 0x4c245c5, 0x10, 0xc005a616c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00fab0510, 0xc00188afa0, 0xc00fab0510, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00fab04b0, 0xc00188afa0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc00fab04b0, 0x4bc7440, 0xc00188afa0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc00fab04b0, 0x4bc7440, 0xc00188afa0, 0xc021c13c20, 0xc021c13c40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00fab04b0, 0x4bc7440, 0xc00188afa0, 0xc005a61990, 0xca72c8, 0x48dd840, 0xc01ad6d700)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc00fab04b0, 0x4bc7440, 0xc00188afa0, 0xc021c13c20, 0xc01ad6d700, 0xc01ad6d700, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00fab04b0, 0x4bc7440, 0xc00188afa0, 0x0, 0xc000100800, 0x0, 0xc005413200)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc00fab04b0, 0x4bc7440, 0xc00188afa0, 0xc021c13c20, 0xc01ad6d6c0, 0xc005a61ac8, 0xca72c8, 0x48dd840, 0xc01ad6d6c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc00fab04b0, 0x4bc7440, 0xc00188afa0, 0xc021c13c20, 0xc021c13c40, 0xc0000adba0, 0x661046, 0x491f620, 0xc00fab04b0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc00fab04b0, 0xc0054131a0, 0xc001681820, 0x5a4e4e0, 0xc00fab04b0, 0xc0236ee000, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc013f2dc00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc013f2dc00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc013f2dc00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216265846 [semacquire, 179 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0104263f0, 0x1688bd7f4675c5c8, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0104263f0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc010426360, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc010426360, 0x4c245c5, 0x10, 0xc003f7d6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc010426360, 0xc01371ea00, 0xc010426360, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc010426060, 0xc01371ea00, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc010426060, 0x4bc7440, 0xc01371ea00, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc010426060, 0x4bc7440, 0xc01371ea00, 0xc00a47d500, 0xc00a47d520, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc010426060, 0x4bc7440, 0xc01371ea00, 0xc003f7d990, 0xca72c8, 0x48dd840, 0xc00f869bc0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc010426060, 0x4bc7440, 0xc01371ea00, 0xc00a47d500, 0xc00f869bc0, 0xc00f869bc0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc010426060, 0x4bc7440, 0xc01371ea00, 0x0, 0xc00009c800, 0x0, 0xc00ff14a80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc010426060, 0x4bc7440, 0xc01371ea00, 0xc00a47d500, 0xc00f869b80, 0xc003f7dac8, 0xca72c8, 0x48dd840, 0xc00f869b80)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc010426060, 0x4bc7440, 0xc01371ea00, 0xc00a47d500, 0xc00a47d520, 0xc001eabba0, 0x661046, 0x491f620, 0xc010426060)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc010426060, 0xc00ff14a20, 0xc001681820, 0x5a4e4e0, 0xc010426060, 0xc003575800, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc013f2d800, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc013f2d800, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc013f2d800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216996422 [chan receive, 32 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc011e3ed00, 0x5a4e4e0, 0xc015820b70, 0xc00ce583e0, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc015820b70, 0x5ab2120, 0xc00e781500, 0xc0093fa028, 0xd, 0xc015820b10, 0x59fb820, 0xc01d5897d0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc015820b70, 0x5ab21e0, 0xc010bde1e0, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc010bde1e0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc010bde1e0, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 217206737 [semacquire, 86 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01d719020, 0x1688c2915676b920, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01d719020, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01d718ff0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01d718ff0, 0x4c245c5, 0x10, 0xc0071c36c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01d718ff0, 0xc012cddea0, 0xc01d718ff0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01d718f90, 0xc012cddea0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01d718f90, 0x4bc7440, 0xc012cddea0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01d718f90, 0x4bc7440, 0xc012cddea0, 0xc0157c0240, 0xc0157c0260, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01d718f90, 0x4bc7440, 0xc012cddea0, 0xc0071c3990, 0xca72c8, 0x48dd840, 0xc0100ce640)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01d718f90, 0x4bc7440, 0xc012cddea0, 0xc0157c0240, 0xc0100ce640, 0xc0100ce640, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01d718f90, 0x4bc7440, 0xc012cddea0, 0x0, 0xc000600000, 0x0, 0xc015c20ba0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01d718f90, 0x4bc7440, 0xc012cddea0, 0xc0157c0240, 0xc0100ce580, 0xc0071c3ac8, 0xca72c8, 0x48dd840, 0xc0100ce580)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01d718f90, 0x4bc7440, 0xc012cddea0, 0xc0157c0240, 0xc0157c0260, 0xc01e93eba0, 0x661046, 0x491f620, 0xc01d718f90)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01d718f90, 0xc015c20ae0, 0xc001681820, 0x5a4e4e0, 0xc01d718f90, 0xc00eb54000, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00393ee00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00393ee00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00393ee00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216094946 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc014d459e0, 0x1688bcc480328c4a, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc014d459e0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc014d459e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc014d459e0, 0x4c245c5, 0x10, 0xc026c58410, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc014d459e0, 0xc00ea38000, 0xc014d459e0, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc014d459b0, 0xc00ea38000, 0xc016a5e180, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc015ddd400, 0x5a4e4e0, 0xc014d459b0, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc015ddd400, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00928f650, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00928f618, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00928f5e8, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00928f548, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00928f548, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00928f478, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00928f458, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00928f3c0, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00928f200, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00e394360, 0x5a4e420, 0xc016a5e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc016a5e180, 0xc026c5d208, 0xc00b452f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc00e394360, 0x5a4e420, 0xc016a5e180, 0xc00b452f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc00e394360, 0x5a4e420, 0xc016a5e180, 0xc00b452f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc024ec6c28, 0x5a4e420, 0xc016a5e180, 0xc01562c600, 0xc00df25de0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc024ec6c00, 0x5a4e420, 0xc016a5e180, 0xc01562c600, 0xc00df25de0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc00a6530e0, 0x5a4e420, 0xc016a5e180, 0xc01562c600, 0xc00df25de0, 0xc016087400, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc00a6530e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc026c5d5a0, 0x4bfc8f0, 0x5, 0x5a50860, 0xc00a6530e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc0160875a0, 0x5a50860, 0xc00a6530e0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc0160875a0, 0x5a50860, 0xc00a6530e0, 0x0, 0xc002759b80)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc016a5e180, 0xc01562c600, 0xc00df25de0, 0x5a50860, 0xc00a6530e0, 0x5a4e420, 0xc016a5e180)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc0042ca000, 0x5a4e420, 0xc016a5e180)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc00b9c6900)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc016087768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc00ff10000, 0x5a4e420, 0xc016a5e180)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc00ff10000, 0x5a4e420, 0xc016a5e180)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc011e365a0, 0x5a4e420, 0xc016a5e180, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc01597ca80, 0xc00e394360, 0xc01fcc1400, 0xc011a64380, 0xc00df25ff8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc0100025d0, 0xc00df25ff8, 0xc01597ca80, 0xc00e394360, 0x5a50860, 0xc00a6530e0, 0x0, 0xc011a64380, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00df25900, 0x5a4e4e0, 0xc0100025d0, 0xc00df25de0, 0x2, 0x7f8b0dd7dcb0, 0xc002a5ea80, 0xc016088000, 0xc00c6468f8, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00df25900, 0x5a4e4e0, 0xc0100025d0, 0xc00df25de0, 0x7f8b0dd7dcb0, 0xc002a5ea80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00df25900, 0x5a4e4e0, 0xc0100025d0, 0x5a6dda0, 0xc01fcc0f00, 0xc005131035, 0x1e9, 0x11, 0x1, 0xc028b81860, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00df25900, 0x5a4e420, 0xc01130aa00, 0x5a6dda0, 0xc01fcc0f00, 0xc005131035, 0x1e9, 0x11, 0x1, 0xc028b81860, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00df25900, 0x5a4e420, 0xc01130aa00, 0xc028b81860, 0xc01ec57d40, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc0051313b4, 0x0, 0x0, 0x30b8551f, 0xed85a80d1, 0x0, 0xc00df25900, 0xc026c5fa20, 0xc026c5fa10, 0x5, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00df25900, 0x5a4e420, 0xc01130aa00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00df25900, 0x5a4e420, 0xc020734f40, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc0086742f0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc020734f40, 0xc00df25900, 0x5400, 0x15000, 0xc001805a40, 0xc0086742f0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc014c261e6, 0xc01da44ba0, 0x5a4e420, 0xc020734f40, 0xc0086742f0, 0xc001a38000, 0xc002a5e400, 0x5ab5540, 0xc01fcc0000, 0xc00fe797a0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 217778520 [semacquire, 29 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc00608a658, 0x1688c5a3a934cd96, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc00608a658, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02318a180, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02318a180, 0x4c245c5, 0x10, 0xc0212ce3f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02318a180, 0xc020599e00, 0xc02318a180, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02318a150, 0xc020599e00, 0xc0052299d8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00670ad00, 0x5a4e4e0, 0xc02318a150, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00670ad00, 0x5a6dfa0, 0xc0052299d8, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc0052299d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc0052299d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc0052299d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc0052299d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc005229838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc01183cd50, 0x5a6dfa0, 0xc005229838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc01183cd18, 0x5a6dfa0, 0xc005229838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc01183cce8, 0x5a6dfa0, 0xc005229838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc01183cc48, 0x5a6dfa0, 0xc005229838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc01183cc48, 0x5a6dfa0, 0xc005229838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc01183cb78, 0x5a6dfa0, 0xc005229838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc01183cb58, 0x5a6dfa0, 0xc005229838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc01183cac0, 0x5a6dfa0, 0xc005229838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc01183c900, 0x5a4e420, 0xc01f3d7100, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc01f3d7100, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc012783e60, 0x5a4e420, 0xc01f3d7100, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc01f3d7100, 0xc0212d31f0, 0xc00b50af00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc012783e60, 0x5a4e420, 0xc01f3d7100, 0xc00b50af00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc012783e60, 0x5a4e420, 0xc01f3d7100, 0xc00b50af00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc0116de628, 0x5a4e420, 0xc01f3d7100, 0xc010ac9500, 0xc015f4d0e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc0116de600, 0x5a4e420, 0xc01f3d7100, 0xc010ac9500, 0xc015f4d0e0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc00a0f3b20, 0x5a4e420, 0xc01f3d7100, 0xc010ac9500, 0xc015f4d0e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc00a0f3b20, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc0212d3588, 0x4bfc8f0, 0x5, 0x5a50860, 0xc00a0f3b20)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc003677588, 0x5a50860, 0xc00a0f3b20, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc003677588, 0x5a50860, 0xc00a0f3b20, 0xc, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc01f3d7100, 0xc010ac9500, 0xc015f4d0e0, 0x5a50860, 0xc00a0f3b20, 0x5a4e420, 0xc01f3d7100)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc0081f5800, 0x5a4e420, 0xc01f3d7100)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc0102e7b00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc003677768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc0159ce000, 0x5a4e420, 0xc01f3d7100)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc0159ce000, 0x5a4e420, 0xc01f3d7100)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc0018ef9e0, 0x5a4e420, 0xc01f3d7100, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc000fb2e00, 0xc012783e60, 0xc013323c80, 0xc005554380, 0xc015f4d2f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc0057e0e78, 0xc015f4d2f8, 0xc000fb2e00, 0xc012783e60, 0x5a50860, 0xc00a0f3b20, 0x0, 0xc005554380, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc015f4cc00, 0x5a6dfa0, 0xc0057e0e78, 0xc015f4d0e0, 0x2, 0x7f8b0dd7dcb0, 0xc00dbbe680, 0xc003678000, 0xc01ebcf498, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc015f4cc00, 0x5a6dfa0, 0xc0057e0e78, 0xc015f4d0e0, 0x7f8b0dd7dcb0, 0xc00dbbe680, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc015f4cc00, 0x5a6dfa0, 0xc0057e0e78, 0x5a6dda0, 0xc013764180, 0xc00b3fd030, 0x1e9, 0x11, 0x1, 0xc014aee0f0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc015f4cc00, 0x5a4e420, 0xc003ed2240, 0x5a6dda0, 0xc013764180, 0xc00b3fd030, 0x1e9, 0x11, 0x1, 0xc014aee0f0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc015f4cc00, 0x5a4e420, 0xc003ed2240, 0xc014aee0f0, 0xc00284e360, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc00b3fd39a, 0x0, 0x0, 0x166b9adc, 0xed85aa6ec, 0x0, 0xc015f4cc00, 0xc0212d5a20, 0xc0212d5a10, 0x5, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc015f4cc00, 0x5a4e420, 0xc003ed2240, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc015f4cc00, 0x5a4e420, 0xc02338c4c0, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc01e01caa0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc02338c4c0, 0xc015f4cc00, 0x5400, 0x15000, 0xc001805a40, 0xc01e01caa0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc0021448e6, 0xc00a652940, 0x5a4e420, 0xc02338c4c0, 0xc01e01caa0, 0xc001a38000, 0xc00dbbe000, 0x5ab5540, 0xc004e84180, 0xc00975ea80, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216095171 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01fc3ac30, 0x1688bcc4cf43aced, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01fc3ac30, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01fc3ac30, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01fc3ac30, 0x4c245c5, 0x10, 0xc026fc4410, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01fc3ac30, 0xc019bf5ae0, 0xc01fc3ac30, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01fc3ac00, 0xc019bf5ae0, 0xc0103acf00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc01700c740, 0x5a4e4e0, 0xc01fc3ac00, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc01700c740, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00ff4b1d0, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00ff4b198, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00ff4b168, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00ff4b0c8, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00ff4b0c8, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00ff4aff8, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00ff4afd8, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00ff4af40, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00ff4ad80, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00cd8d050, 0x5a4e420, 0xc0103acf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc0103acf00, 0xc026fc9208, 0xc0208a9900, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc00cd8d050, 0x5a4e420, 0xc0103acf00, 0xc0208a9900, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc00cd8d050, 0x5a4e420, 0xc0103acf00, 0xc0208a9900, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc012d4e028, 0x5a4e420, 0xc0103acf00, 0xc01a068000, 0xc00d410ae0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc012d4e000, 0x5a4e420, 0xc0103acf00, 0xc01a068000, 0xc00d410ae0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc0213feac0, 0x5a4e420, 0xc0103acf00, 0xc01a068000, 0xc00d410ae0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc0213feac0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc026fc95a0, 0x4bfc8f0, 0x5, 0x5a50860, 0xc0213feac0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc026fc95a0, 0x5a50860, 0xc0213feac0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc026fc95a0, 0x5a50860, 0xc0213feac0, 0x0, 0xc0233220a0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc0103acf00, 0xc01a068000, 0xc00d410ae0, 0x5a50860, 0xc0213feac0, 0x5a4e420, 0xc0103acf00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc011d4cc00, 0x5a4e420, 0xc0103acf00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc00bc48000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc026fc9768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc00aebc000, 0x5a4e420, 0xc0103acf00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc00aebc000, 0x5a4e420, 0xc0103acf00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00cbaf200, 0x5a4e420, 0xc0103acf00, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc0010aa0e0, 0xc00cd8d050, 0xc00f8ca880, 0xc01505c000, 0xc00d410cf8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc022018990, 0xc00d410cf8, 0xc0010aa0e0, 0xc00cd8d050, 0x5a50860, 0xc0213feac0, 0x0, 0xc01505c000, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00d410600, 0x5a4e4e0, 0xc022018990, 0xc00d410ae0, 0x2, 0x7f8b0dd7dcb0, 0xc002a21c80, 0xc026fca000, 0xc008c6c4d8, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00d410600, 0x5a4e4e0, 0xc022018990, 0xc00d410ae0, 0x7f8b0dd7dcb0, 0xc002a21c80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00d410600, 0x5a4e4e0, 0xc022018990, 0x5a6dda0, 0xc011af8380, 0xc00ba54015, 0x1e9, 0x11, 0x1, 0xc00a978c30, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00d410600, 0x5a4e420, 0xc00dab6a40, 0x5a6dda0, 0xc011af8380, 0xc00ba54015, 0x1e9, 0x11, 0x1, 0xc00a978c30, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00d410600, 0x5a4e420, 0xc00dab6a40, 0xc00a978c30, 0xc00cbae5a0, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc00ba54381, 0x0, 0x0, 0x89438f6, 0xed85a80d3, 0x0, 0xc00d410600, 0xc026fcba20, 0xc026fcba10, 0x7d, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00d410600, 0x5a4e420, 0xc00dab6a40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00d410600, 0x5a4e420, 0xc0109af480, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc00e356050, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc0109af480, 0xc00d410600, 0x5400, 0x15000, 0xc001805a40, 0xc00e356050, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc0123b3781, 0xc00a17e660, 0x5a4e420, 0xc0109af480, 0xc00e356050, 0xc001a38000, 0xc002a21600, 0x5ab5540, 0xc01e53c100, 0xc016b08ae0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216085532 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).GetRangeInfo(0xc019f59200, 0x5a4e4e0, 0xc021fffda0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica.go:881 +0x4a8
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).maybeAddRangeInfoToResponse(0xc019f59200, 0x5a4e4e0, 0xc021fffda0, 0xc010ff5ae0, 0xc00879e780)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:227 +0x67
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc019f59200, 0x5a4e4e0, 0xc021fffb90, 0x5c79, 0xc010ff5ae0, 0x0, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:123 +0x3aa
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:34
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc021fffb00, 0x1688bcbdf511fc39, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:195 +0x5cc
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc021fffb00, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc021fffb00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc021fffb00, 0x4c245c5, 0x10, 0xc019c46880, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc021fffb00, 0xc010ff5a40, 0xc021fffb00, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc021fffaa0, 0xc010ff5a40, 0xc014844580, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00fdc41c0, 0x5a4e4e0, 0xc021fffaa0, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00fdc41c0, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1145 +0x3e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:784 +0x5e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0196451d0, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc019645198, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc019645168, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:126 +0x8db
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc0196450c8, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc0196450c8, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc019644ff8, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc019644fd8, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc019644f40, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc019644d80, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00f7ca990, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcher.func1(0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:217 +0x85
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc0087dc0e0, 0x5a4e420, 0xc014844580, 0x0, 0x6bebe)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:379 +0x739
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc0087dc0e0, 0x5a4e420, 0xc014844580, 0x7, 0x8, 0x0, 0x0, 0x39, 0x6bf62, 0xffffffffffffffff, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:480 +0x1ab
github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc010b4e750, 0x5a4e420, 0xc014844580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:129 +0x104
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).nextBatch(0xc00000cf00, 0x5a4e420, 0xc014844580, 0xc014fb7380, 0x11044, 0xe9ddc, 0x27)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:803 +0xb9
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:785
github.com/cockroachdb/cockroach/pkg/sql/colmem.(*Allocator).PerformOperation(0xc021ffe450, 0xc006f12a00, 0x28, 0x28, 0xc019c4bbb0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colmem/allocator.go:300 +0x59
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc00000cf00, 0x5a4e420, 0xc014844580, 0x0, 0xc014844500, 0x0, 0xc019c4bc38)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:777 +0xa5
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc004c99300, 0x5a4e420, 0xc014844580, 0xc021ffe840, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x87
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc021ffe4b0, 0x5a4e420, 0xc014844580, 0xc021ffe840, 0xc019c4bd00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:48 +0x69
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc020d02aa0, 0x5a4e420, 0xc014844580, 0x20, 0x5a69060)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selEQInt64Int64ConstOp).Next(0xc007a7f8c0, 0x5a4e420, 0xc014844580, 0x5b8a880, 0xc019c4bdf0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:6114 +0xf7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selGEInt64Int64ConstOp).Next(0xc007a7f9e0, 0x5a4e420, 0xc014844580, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:54994 +0xf7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selGTTimestampTimestampConstOp).Next(0xc010ff45a0, 0x5a4e420, 0xc014844580, 0xc019c4bf80, 0x46164c)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:48146 +0x237
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc00b337860, 0x5a4e420, 0xc014844580, 0xc028b88fe0, 0x7f89bc840328)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/is_null_ops.eg.go:241 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BatchSchemaSubsetEnforcer).Next(0xc010f7d4a0, 0x5a4e420, 0xc014844580, 0xc003d76000, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:212 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*bufferOp).advance(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/buffer.go:62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*caseOp).Next(0xc007a7fe60, 0x5a4e420, 0xc014844580, 0x400, 0xc014fb7540)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/case.go:109 +0x6b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.selBoolOp.Next(0x5a518a0, 0xc007a7fe60, 0x0, 0x0, 0xc014844900, 0xf, 0x5a4e420, 0xc014844580, 0x7fffffffffffffff, 0xc014fb7500)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/bool_vec_to_sel.go:149 +0x55
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BoolVecToSelOp).Next(0xc014844900, 0x5a4e420, 0xc014844580, 0xc014fb6500, 0xc00c089400)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/bool_vec_to_sel.go:58 +0x82
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*selectInOpBytes).Next(0xc014844940, 0x5a4e420, 0xc014844580, 0xc014fb6cc0, 0xc016956000)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/select_in.eg.go:639 +0x1f0
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*selectInOpBytes).Next(0xc014844980, 0x5a4e420, 0xc014844580, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/select_in.eg.go:639 +0x1f0
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selNEBytesBytesConstOp).Next(0xc010ff4aa0, 0x5a4e420, 0xc014844580, 0x1c6f445, 0xc019c4c710)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:10156 +0x1e8
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc020d02af0, 0x5a4e420, 0xc014844580, 0xbc, 0x7f8b00d00a18)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc020d02b90, 0x5a4e420, 0xc014844580, 0x19d8503, 0xc00a9471a0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc00fdd0000, 0x19dad57, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:290 +0x163
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:315
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc019c4c758, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc00fdd0000, 0xc0104dd380, 0x1, 0xc021ffe9f0, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:321 +0x5b
github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput(0xc009e0e000, 0x10, 0x10, 0x1, 0x5b8a680, 0xc019c4c8c8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:562 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next(0xc009e0e000, 0x1, 0x10, 0xc008d58600, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:517 +0x1bb
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc012cee900, 0x5a4e420, 0xc014844580, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:199 +0x10b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc01a4f2f20, 0x5a4e420, 0xc014844580, 0xc023d30400, 0xc021fff410)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/is_null_ops.eg.go:241 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc014844a00, 0x5a4e420, 0xc014844580, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.constBoolOp.Next(0x5a69060, 0xc014844a00, 0xc021ffeb40, 0x10, 0x0, 0x5a4e420, 0xc014844580, 0xc010b4e750, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/const.eg.go:164 +0x63
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc014844a40, 0x5a4e420, 0xc014844580, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*defaultBuiltinFuncOperator).Next(0xc010b4e360, 0x5a4e420, 0xc014844580, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/builtin_funcs.go:48 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc020d02dc0, 0x5a4e420, 0xc014844580, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*allSpooler).spool(0xc010f7d8c0, 0x5a4e420, 0xc014844580)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:140 +0x5f
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*sortOp).Next(0xc012c21380, 0x5a4e420, 0xc014844580, 0x7c8fe01, 0xc004c99340)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:261 +0xbf
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc019c4d018, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc00879e400, 0x5a4e420, 0xc014844580, 0xc010f7d4a0, 0xc019c4d088)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*unorderedDistinct).Next(0xc021ffeea0, 0x5a4e420, 0xc014844580, 0x5a69060, 0xc014844680)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/unordered_distinct.go:72 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc019c4d158, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc00879e500, 0x5a4e420, 0xc014844580, 0x2a09593, 0xc021ffe570)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc014845380, 0x5a4e420, 0xc014844580, 0x0, 0xc019c4d228)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*defaultBuiltinFuncOperator).Next(0xc010b4e630, 0x5a4e420, 0xc014844580, 0x203006, 0x203006)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/builtin_funcs.go:48 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc020d02f50, 0x5a4e420, 0xc014844580, 0x20300200000000, 0x7f8afe7eefff)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).spool(0xc010185000, 0x5a4e420, 0xc014844580)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sorttopk.go:147 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).Next(0xc010185000, 0x5a4e420, 0xc014844580, 0xc019c4d4d0, 0x4616aa)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sorttopk.go:118 +0xee
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc019c4d550, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc00879e580, 0x5a4e420, 0xc014844580, 0xc019c4d610, 0x2906f85)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc020d03040, 0x5a4e420, 0xc014844580, 0xc01a9a86c0, 0x6)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*limitOp).Next(0xc021fff380, 0x5a4e420, 0xc014844580, 0xc019c4d6a8, 0x2a238d3)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/limit.go:53 +0x55
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc0060e4000, 0xc020d03040, 0xc019c4d6d8, 0x2d229ef)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:290 +0x163
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:315
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc019c4d730, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc0060e4000, 0x5a4e420, 0xc014844580, 0x2d229c0, 0x5a51b20)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:321 +0x5b
github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run(0x5a4e420, 0xc014844580, 0x5a745e0, 0xc0060e4000, 0x5a16ca0, 0xc00c21c700)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:171 +0x35
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc0060e4000, 0x5a4e420, 0xc014844580)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:781 +0x96
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc011567b00, 0x5a4e420, 0xc014844580, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc019681a40, 0xc00f7ca990, 0xc003fb7400, 0xc00c21c700, 0xc015c6e6f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc028ad7470, 0xc015c6e6f8, 0xc019681a40, 0xc00f7ca990, 0x5a507e0, 0xc010ff41e0, 0x0, 0xc00c21c700, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc015c6e000, 0x5a4e4e0, 0xc028ad7470, 0xc015c6e4e0, 0x3, 0x7f8b0dd7dcb0, 0xc00ddd2a80, 0xc019c4e001, 0xc02507a958, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc015c6e000, 0x5a4e4e0, 0xc028ad7470, 0xc015c6e4e0, 0x7f8b0dd7dcb0, 0xc00ddd2a80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc015c6e000, 0x5a4e4e0, 0xc028ad7470, 0x5a6ce20, 0xc00d4db270, 0xc006c96018, 0x269, 0xb, 0x1, 0xc025b172c0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216085793 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc015b330e0, 0x1688bcbdf92cc54c, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc015b330e0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc015b330e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc015b330e0, 0x4c245c5, 0x10, 0xc01eb40880, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc015b330e0, 0xc0141b6f00, 0xc015b330e0, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc015b330b0, 0xc0141b6f00, 0xc00422e3c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00ed7c400, 0x5a4e4e0, 0xc015b330b0, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00ed7c400, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1145 +0x3e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:784 +0x5e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0123edad0, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc0123eda98, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc0123eda68, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:126 +0x8db
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc0123ed9c8, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc0123ed9c8, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc0123ed8f8, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc0123ed8d8, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0123ed840, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc0123ed680, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00bf91a70, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcher.func1(0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:217 +0x85
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc0005eddc0, 0x5a4e420, 0xc00422e3c0, 0x0, 0x6bebe)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:379 +0x739
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc0005eddc0, 0x5a4e420, 0xc00422e3c0, 0x7, 0x8, 0x0, 0x0, 0x39, 0x6bf62, 0xffffffffffffffff, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:480 +0x1ab
github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc00959f050, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:129 +0x104
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).nextBatch(0xc010c7ba40, 0x5a4e420, 0xc00422e3c0, 0xc01a475a80, 0x11044, 0xe9ddc, 0x27)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:803 +0xb9
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:785
github.com/cockroachdb/cockroach/pkg/sql/colmem.(*Allocator).PerformOperation(0xc00cf85590, 0xc004901680, 0x28, 0x28, 0xc01eb45bb0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colmem/allocator.go:300 +0x59
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc010c7ba40, 0x5a4e420, 0xc00422e3c0, 0x0, 0xc00422e300, 0x0, 0xc01eb45c38)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:777 +0xa5
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc0070a1480, 0x5a4e420, 0xc00422e3c0, 0xc00cf85a70, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x87
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc00cf855f0, 0x5a4e420, 0xc00422e3c0, 0xc00cf85a70, 0xc01eb45d00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:48 +0x69
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc002cb8d20, 0x5a4e420, 0xc00422e3c0, 0x20, 0x5a69060)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selEQInt64Int64ConstOp).Next(0xc00959e090, 0x5a4e420, 0xc00422e3c0, 0x5b8a880, 0xc01eb45df0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:6114 +0xf7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selGEInt64Int64ConstOp).Next(0xc00959e1b0, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:54994 +0xf7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selGTTimestampTimestampConstOp).Next(0xc001b1a5a0, 0x5a4e420, 0xc00422e3c0, 0xc01eb45f80, 0x46164c)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:48146 +0x237
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc00b304be0, 0x5a4e420, 0xc00422e3c0, 0xc0054cafe0, 0x7f8b0dcb21d8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/is_null_ops.eg.go:241 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BatchSchemaSubsetEnforcer).Next(0xc011d39b60, 0x5a4e420, 0xc00422e3c0, 0xc0002a9800, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:212 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*bufferOp).advance(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/buffer.go:62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*caseOp).Next(0xc00959e630, 0x5a4e420, 0xc00422e3c0, 0x400, 0xc01a475c00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/case.go:109 +0x6b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.selBoolOp.Next(0x5a518a0, 0xc00959e630, 0x0, 0x0, 0xc00422ea00, 0xf, 0x5a4e420, 0xc00422e3c0, 0x7fffffffffffffff, 0xc01a475c00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/bool_vec_to_sel.go:149 +0x55
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BoolVecToSelOp).Next(0xc00422ea00, 0x5a4e420, 0xc00422e3c0, 0xc01a474c80, 0xc0074fe800)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/bool_vec_to_sel.go:58 +0x82
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*selectInOpBytes).Next(0xc00422ea40, 0x5a4e420, 0xc00422e3c0, 0xc01a475340, 0xc0074ff400)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/select_in.eg.go:639 +0x1f0
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*selectInOpBytes).Next(0xc00422ea80, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/select_in.eg.go:639 +0x1f0
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selNEBytesBytesConstOp).Next(0xc001b1b860, 0x5a4e420, 0xc00422e3c0, 0x1c6f445, 0xc01eb46710)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel/selection_ops.eg.go:10156 +0x1e8
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc002cb8d70, 0x5a4e420, 0xc00422e3c0, 0xbb, 0x7f89bc988ad8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc002cb8dc0, 0x5a4e420, 0xc00422e3c0, 0x19d8503, 0xc00c01a9b0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc0088e1800, 0x19dad57, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:290 +0x163
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:315
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01eb46758, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc0088e1800, 0xc010122660, 0x1, 0xc00cf85ec0, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:321 +0x5b
github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput(0xc02254cc00, 0x10, 0x10, 0x1, 0x5b8a680, 0xc01eb468c8)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:562 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next(0xc02254cc00, 0x1, 0x10, 0xc00dd10300, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:517 +0x1bb
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc0249ba000, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:199 +0x10b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc01677e2e0, 0x5a4e420, 0xc00422e3c0, 0xc00e04fe70, 0xc015b32a50)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/is_null_ops.eg.go:241 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc00422ebc0, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.constBoolOp.Next(0x5a69060, 0xc00422ebc0, 0xc015b320c0, 0x10, 0x0, 0x5a4e420, 0xc00422e3c0, 0xc00959f050, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/const.eg.go:164 +0x63
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc00422ec00, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*defaultBuiltinFuncOperator).Next(0xc00959eab0, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/builtin_funcs.go:48 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc002cb9a40, 0x5a4e420, 0xc00422e3c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*allSpooler).spool(0xc01974a000, 0x5a4e420, 0xc00422e3c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:140 +0x5f
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*sortOp).Next(0xc025245500, 0x5a4e420, 0xc00422e3c0, 0x7c8fe01, 0xc0070a14c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:261 +0xbf
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01eb47018, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc014bcfb80, 0x5a4e420, 0xc00422e3c0, 0xc011d39b60, 0xc01eb47088)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*unorderedDistinct).Next(0xc015b32540, 0x5a4e420, 0xc00422e3c0, 0x5a69060, 0xc00422e480)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/unordered_distinct.go:72 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01eb47158, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc014bcff00, 0x5a4e420, 0xc00422e3c0, 0x2a09593, 0xc00cf856b0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc00422fc80, 0x5a4e420, 0xc00422e3c0, 0x0, 0xae9a)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:145 +0x4b
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*defaultBuiltinFuncOperator).Next(0xc00959eea0, 0x5a4e420, 0xc00422e3c0, 0x203005, 0x203005)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/builtin_funcs.go:48 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc002cb9d10, 0x5a4e420, 0xc00422e3c0, 0x20300200000000, 0x7f8afe7eefff)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).spool(0xc008842400, 0x5a4e420, 0xc00422e3c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sorttopk.go:147 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).Next(0xc008842400, 0x5a4e420, 0xc00422e3c0, 0xc01eb474d0, 0x4616aa)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/sorttopk.go:118 +0xee
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:201 +0x4e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01eb47550, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc014bcff80, 0x5a4e420, 0xc00422e3c0, 0xc01eb47610, 0x2906f85)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:199 +0xa7
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc002cb9e00, 0x5a4e420, 0xc00422e3c0, 0xc01974aba0, 0x6)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*limitOp).Next(0xc015b329f0, 0x5a4e420, 0xc00422e3c0, 0xc01eb476a8, 0x2a238d3)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/limit.go:53 +0x55
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc01081cc00, 0xc002cb9e00, 0xc01eb476d8, 0x2d229ef)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:290 +0x163
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:315
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01eb47730, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc01081cc00, 0x5a4e420, 0xc00422e3c0, 0x2d229c0, 0x5a51b20)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:321 +0x5b
github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run(0x5a4e420, 0xc00422e3c0, 0x5a745e0, 0xc01081cc00, 0x5a16ca0, 0xc00dd93880)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:171 +0x35
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc01081cc00, 0x5a4e420, 0xc00422e3c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:781 +0x96
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00551c6c0, 0x5a4e420, 0xc00422e3c0, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc0005ed6c0, 0xc00bf91a70, 0xc014bcf080, 0xc00dd93880, 0xc015e232f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc0190fae70, 0xc015e232f8, 0xc0005ed6c0, 0xc00bf91a70, 0x5a507e0, 0xc002c0be00, 0x0, 0xc00dd93880, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc015e22c00, 0x5a4e4e0, 0xc0190fae70, 0xc015e230e0, 0x3, 0x7f8b0dd7dcb0, 0xc002334a80, 0xc01eb48001, 0xc011d384d8, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc015e22c00, 0x5a4e4e0, 0xc0190fae70, 0xc015e230e0, 0x7f8b0dd7dcb0, 0xc002334a80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc015e22c00, 0x5a4e4e0, 0xc0190fae70, 0x5a6ce20, 0xc00f6b7f40, 0xc0086a7018, 0x269, 0xb, 0x1, 0xc02ae5b770, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc015e22c00, 0x5a4e420, 0xc01f2e5080, 0x5a6ce20, 0xc00f6b7f40, 0xc0086a7018, 0x269, 0xb, 0x1, 0xc02ae5b770, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc015e22c00, 0x5a4e420, 0xc01f2e5080, 0xc02ae5b770, 0xc009ad06e0, 0xb, 0xb, 0xc006ff2820, 0x5, 0x5, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 217409681 [semacquire, 67 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0081f0120, 0x1688c393c8e3e75b, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0081f0120, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0081f0090, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0081f0090, 0x4c245c5, 0x10, 0xc0041a96c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0081f0090, 0xc023055680, 0xc0081f0090, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0138f3410, 0xc023055680, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0138f3410, 0x4bc7440, 0xc023055680, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0138f3410, 0x4bc7440, 0xc023055680, 0xc0151ebce0, 0xc0151ebd00, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0138f3410, 0x4bc7440, 0xc023055680, 0xc0041a9990, 0xca72c8, 0x48dd840, 0xc012aed580)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0138f3410, 0x4bc7440, 0xc023055680, 0xc0151ebce0, 0xc012aed580, 0xc012aed580, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0138f3410, 0x4bc7440, 0xc023055680, 0x0, 0xc00758a000, 0x0, 0xc019fd9f20)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0138f3410, 0x4bc7440, 0xc023055680, 0xc0151ebce0, 0xc012aed540, 0xc0041a9ac8, 0xca72c8, 0x48dd840, 0xc012aed540)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0138f3410, 0x4bc7440, 0xc023055680, 0xc0151ebce0, 0xc0151ebd00, 0xc0013f0ba0, 0x661046, 0x491f620, 0xc0138f3410)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0138f3410, 0xc019fd9ec0, 0xc001681820, 0x5a4e4e0, 0xc0138f3410, 0xc01f9acc00, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0061b8900, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0061b8900, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0061b8900)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216416091 [semacquire, 165 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02064acc0, 0x1688be3bdf2a6d92, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02064acc0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02064ac90, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02064ac90, 0x4c245c5, 0x10, 0xc00394b6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02064ac90, 0xc002a0d7c0, 0xc02064ac90, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02064a510, 0xc002a0d7c0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02064a510, 0x4bc7440, 0xc002a0d7c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02064a510, 0x4bc7440, 0xc002a0d7c0, 0xc015f3f4e0, 0xc015f3f500, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02064a510, 0x4bc7440, 0xc002a0d7c0, 0xc00394b990, 0xca72c8, 0x48dd840, 0xc02313ce00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02064a510, 0x4bc7440, 0xc002a0d7c0, 0xc015f3f4e0, 0xc02313ce00, 0xc02313ce00, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02064a510, 0x4bc7440, 0xc002a0d7c0, 0x0, 0xc00fba5800, 0x0, 0xc00f70d1a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02064a510, 0x4bc7440, 0xc002a0d7c0, 0xc015f3f4e0, 0xc02313cdc0, 0xc00394bac8, 0xca72c8, 0x48dd840, 0xc02313cdc0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02064a510, 0x4bc7440, 0xc002a0d7c0, 0xc015f3f4e0, 0xc015f3f500, 0xc0057b4ba0, 0x661046, 0x491f620, 0xc02064a510)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02064a510, 0xc00f70d140, 0xc001681820, 0x5a4e4e0, 0xc02064a510, 0xc004333800, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc023342100, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc023342100, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc023342100)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216095540 [chan receive, 105 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc011577600, 0x5a4e4e0, 0xc012cb4ff0, 0xc00b06a0a0, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc012cb4ff0, 0x5ab2120, 0xc013a42e00, 0xc00bf1d028, 0xd, 0xc012cb4f90, 0x59fb820, 0xc01f4acb70, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc012cb4ff0, 0x5ab21e0, 0xc00cadf2c0, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc00cadf2c0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc00cadf2c0, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 216218340 [semacquire, 183 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc002e3e0f0, 0x1688bd43e702326b, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc002e3e0f0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc002e3e090, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc002e3e090, 0x4c245c5, 0x10, 0xc0040a36c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc002e3e090, 0xc002478d20, 0xc002e3e090, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0201e9f20, 0xc002478d20, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0201e9f20, 0x4bc7440, 0xc002478d20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0201e9f20, 0x4bc7440, 0xc002478d20, 0xc01023ede0, 0xc01023ee20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0201e9f20, 0x4bc7440, 0xc002478d20, 0xc0040a3990, 0xca72c8, 0x48dd840, 0xc010476e40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0201e9f20, 0x4bc7440, 0xc002478d20, 0xc01023ede0, 0xc010476e40, 0xc010476e40, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0201e9f20, 0x4bc7440, 0xc002478d20, 0x0, 0xc00fba5800, 0x0, 0xc00aeae300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0201e9f20, 0x4bc7440, 0xc002478d20, 0xc01023ede0, 0xc010476dc0, 0xc0040a3ac8, 0xca72c8, 0x48dd840, 0xc010476dc0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0201e9f20, 0x4bc7440, 0xc002478d20, 0xc01023ede0, 0xc01023ee20, 0xc00f81fba0, 0x661046, 0x491f620, 0xc0201e9f20)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0201e9f20, 0xc00aeae2a0, 0xc001681820, 0x5a4e4e0, 0xc0201e9f20, 0xc00cb9f200, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01b41f000, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01b41f000, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01b41f000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217780418 [semacquire, 29 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01fdbba10, 0x1688c5a59622e079, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01fdbba10, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01fdbb9e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01fdbb9e0, 0x4c245c5, 0x10, 0xc00be796c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01fdbb9e0, 0xc00bb3eaa0, 0xc01fdbb9e0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01fdbb980, 0xc00bb3eaa0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01fdbb980, 0x4bc7440, 0xc00bb3eaa0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01fdbb980, 0x4bc7440, 0xc00bb3eaa0, 0xc01e6c56c0, 0xc01e6c56e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01fdbb980, 0x4bc7440, 0xc00bb3eaa0, 0xc00be79990, 0xca72c8, 0x48dd840, 0xc021cf0ec0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01fdbb980, 0x4bc7440, 0xc00bb3eaa0, 0xc01e6c56c0, 0xc021cf0ec0, 0xc021cf0ec0, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01fdbb980, 0x4bc7440, 0xc00bb3eaa0, 0x0, 0xc003d76000, 0x0, 0xc008a8b680)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01fdbb980, 0x4bc7440, 0xc00bb3eaa0, 0xc01e6c56c0, 0xc021cf0e80, 0xc00be79ac8, 0xca72c8, 0x48dd840, 0xc021cf0e80)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01fdbb980, 0x4bc7440, 0xc00bb3eaa0, 0xc01e6c56c0, 0xc01e6c56e0, 0xc00187dba0, 0x661046, 0x491f620, 0xc01fdbb980)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01fdbb980, 0xc008a8b620, 0xc001681820, 0x5a4e4e0, 0xc01fdbb980, 0xc00be1c000, 0x336)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00ef3dc00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00ef3dc00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00ef3dc00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216094720 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc008f92690, 0x1688bcc46b1870b0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc008f92690, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc008f92660, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc008f92660, 0x4c245c5, 0x10, 0xc0073436c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc008f92660, 0xc0141b79a0, 0xc008f92660, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc008f92600, 0xc0141b79a0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc008f92600, 0x4bc7440, 0xc0141b79a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc008f92600, 0x4bc7440, 0xc0141b79a0, 0xc00b416d60, 0xc00b416d80, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc008f92600, 0x4bc7440, 0xc0141b79a0, 0xc007343990, 0xca72c8, 0x48dd840, 0xc0218313c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc008f92600, 0x4bc7440, 0xc0141b79a0, 0xc00b416d60, 0xc0218313c0, 0xc0218313c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc008f92600, 0x4bc7440, 0xc0141b79a0, 0x0, 0x87067e0, 0x0, 0xc0159c5620)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc008f92600, 0x4bc7440, 0xc0141b79a0, 0xc00b416d60, 0xc021831380, 0xc007343ac8, 0xca72c8, 0x48dd840, 0xc021831380)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc008f92600, 0x4bc7440, 0xc0141b79a0, 0xc00b416d60, 0xc00b416d80, 0xc0020f4ba0, 0x661046, 0x491f620, 0xc008f92600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc008f92600, 0xc0159c55c0, 0xc001681820, 0x5a4e4e0, 0xc008f92600, 0xc006957200, 0x32f)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f091200, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f091200, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f091200)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216997371 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc002367500, 0x1688c18c684f0e50, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc002367500, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc002367500, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc002367500, 0x4c245c5, 0x10, 0xc02c666410, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc002367500, 0xc0113bd860, 0xc002367500, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0023674d0, 0xc0113bd860, 0xc011c9fe40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc01ed97000, 0x5a4e4e0, 0xc0023674d0, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc01ed97000, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0027df1d0, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc0027df198, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc0027df168, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc0027df0c8, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc0027df0c8, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc0027deff8, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc0027defd8, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0027def40, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc0027ded80, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc01388a480, 0x5a4e420, 0xc011c9fe40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc011c9fe40, 0xc02c66b208, 0xc00f045400, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc01388a480, 0x5a4e420, 0xc011c9fe40, 0xc00f045400, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc01388a480, 0x5a4e420, 0xc011c9fe40, 0xc00f045400, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc00f351228, 0x5a4e420, 0xc011c9fe40, 0xc00ca61200, 0xc00d220ae0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00f351200, 0x5a4e420, 0xc011c9fe40, 0xc00ca61200, 0xc00d220ae0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc00a1bc460, 0x5a4e420, 0xc011c9fe40, 0xc00ca61200, 0xc00d220ae0, 0xc02c66b400, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc00a1bc460, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc02c66b5a0, 0x4bfc8f0, 0x5, 0x5a50860, 0xc00a1bc460)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc02c66b5a0, 0x5a50860, 0xc00a1bc460, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc02c66b5a0, 0x5a50860, 0xc00a1bc460, 0x0, 0xc015814be0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc011c9fe40, 0xc00ca61200, 0xc00d220ae0, 0x5a50860, 0xc00a1bc460, 0x5a4e420, 0xc011c9fe40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc00f351800, 0x5a4e420, 0xc011c9fe40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc002f90400)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc02c66b768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc0147fb800, 0x5a4e420, 0xc011c9fe40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc0147fb800, 0x5a4e420, 0xc011c9fe40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00badbc20, 0x5a4e420, 0xc011c9fe40, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc00df35340, 0xc01388a480, 0xc01f264d80, 0xc005898380, 0xc00d220cf8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a4e4e0, 0xc01e33d1d0, 0xc00d220cf8, 0xc00df35340, 0xc01388a480, 0x5a50860, 0xc00a1bc460, 0x0, 0xc005898380, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00d220600, 0x5a4e4e0, 0xc01e33d1d0, 0xc00d220ae0, 0x2, 0x7f8b0dd7dcb0, 0xc00258d380, 0xc02c66c000, 0xc00ff1e778, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00d220600, 0x5a4e4e0, 0xc01e33d1d0, 0xc00d220ae0, 0x7f8b0dd7dcb0, 0xc00258d380, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00d220600, 0x5a4e4e0, 0xc01e33d1d0, 0x5a6dda0, 0xc00b551d80, 0xc019ffb0e5, 0x1e9, 0x11, 0x1, 0xc00d1080f0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00d220600, 0x5a4e420, 0xc010f1bec0, 0x5a6dda0, 0xc00b551d80, 0xc019ffb0e5, 0x1e9, 0x11, 0x1, 0xc00d1080f0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00d220600, 0x5a4e420, 0xc010f1bec0, 0xc00d1080f0, 0xc00badb320, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc019ffb462, 0x0, 0x0, 0x1b376ee1, 0xed85a955a, 0x0, 0xc00d220600, 0xc02c66da20, 0xc02c66da10, 0xac, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00d220600, 0x5a4e420, 0xc010f1bec0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00d220600, 0x5a4e420, 0xc019b68440, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc02357d020, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc019b68440, 0xc00d220600, 0x5400, 0x15000, 0xc001805a40, 0xc02357d020, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc00bdc5266, 0xc01640c100, 0x5a4e420, 0xc019b68440, 0xc02357d020, 0xc001a38000, 0xc00258cd00, 0x5ab5540, 0xc01304b700, 0xc0225631a0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 218081989 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0298fbe60, 0x1688c72a292e6a05, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0298fbe60, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0298fbe30, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0298fbe30, 0x4c245c5, 0x10, 0xc02a6a16c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0298fbe30, 0xc0296f90e0, 0xc0298fbe30, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0298fbda0, 0xc0296f90e0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0298fbda0, 0x4bc7440, 0xc0296f90e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0298fbda0, 0x4bc7440, 0xc0296f90e0, 0xc0168ca8c0, 0xc0168ca920, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0298fbda0, 0x4bc7440, 0xc0296f90e0, 0xc02a6a1990, 0xca72c8, 0x48dd840, 0xc00c6b45c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0298fbda0, 0x4bc7440, 0xc0296f90e0, 0xc0168ca8c0, 0xc00c6b45c0, 0xc00c6b45c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0298fbda0, 0x4bc7440, 0xc0296f90e0, 0x0, 0xc00e8e4800, 0x0, 0xc029e86ea0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0298fbda0, 0x4bc7440, 0xc0296f90e0, 0xc0168ca8c0, 0xc00c6b4580, 0xc02a6a1ac8, 0xca72c8, 0x48dd840, 0xc00c6b4580)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0298fbda0, 0x4bc7440, 0xc0296f90e0, 0xc0168ca8c0, 0xc0168ca920, 0xc002294ba0, 0x661046, 0x491f620, 0xc0298fbda0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0298fbda0, 0xc029e86e40, 0xc001681820, 0x5a4e4e0, 0xc0298fbda0, 0xc029ea6c00, 0x303)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029eb2000, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029eb2000, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029eb2000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217782699 [semacquire, 29 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01b11eba0, 0x1688c5a7f0e6fc1b, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01b11eba0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01b11eb70, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01b11eb70, 0x4c245c5, 0x10, 0xc00c9a96c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01b11eb70, 0xc01520f2c0, 0xc01b11eb70, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01b11eb10, 0xc01520f2c0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01b11eb10, 0x4bc7440, 0xc01520f2c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01b11eb10, 0x4bc7440, 0xc01520f2c0, 0xc021e26e60, 0xc021e26e80, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01b11eb10, 0x4bc7440, 0xc01520f2c0, 0xc00c9a9990, 0xca72c8, 0x48dd840, 0xc014656a40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01b11eb10, 0x4bc7440, 0xc01520f2c0, 0xc021e26e60, 0xc014656a40, 0xc014656a40, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01b11eb10, 0x4bc7440, 0xc01520f2c0, 0x0, 0x87067e0, 0x0, 0xc022f77860)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01b11eb10, 0x4bc7440, 0xc01520f2c0, 0xc021e26e60, 0xc014656a00, 0xc00c9a9ac8, 0xca72c8, 0x48dd840, 0xc014656a00)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01b11eb10, 0x4bc7440, 0xc01520f2c0, 0xc021e26e60, 0xc021e26e80, 0xc0072a5ba0, 0x661046, 0x491f620, 0xc01b11eb10)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01b11eb10, 0xc022f77800, 0xc001681820, 0x5a4e4e0, 0xc01b11eb10, 0xc0098e2000, 0x2f3)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007053100, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007053100, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007053100)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217341662 [semacquire, 73 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01df70030, 0x1688c33da2c06d0b, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01df70030, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01df70000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01df70000, 0x4c245c5, 0x10, 0xc00516f6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01df70000, 0xc002f4da40, 0xc01df70000, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc002b01e90, 0xc002f4da40, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc002b01e90, 0x4bc7440, 0xc002f4da40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc002b01e90, 0x4bc7440, 0xc002f4da40, 0xc01f637860, 0xc01f637880, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc002b01e90, 0x4bc7440, 0xc002f4da40, 0xc00516f990, 0xca72c8, 0x48dd840, 0xc001a4c240)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc002b01e90, 0x4bc7440, 0xc002f4da40, 0xc01f637860, 0xc001a4c240, 0xc001a4c240, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc002b01e90, 0x4bc7440, 0xc002f4da40, 0x0, 0x87067e0, 0x0, 0xc01558a5a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc002b01e90, 0x4bc7440, 0xc002f4da40, 0xc01f637860, 0xc001a4c200, 0xc00516fac8, 0xca72c8, 0x48dd840, 0xc001a4c200)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc002b01e90, 0x4bc7440, 0xc002f4da40, 0xc01f637860, 0xc01f637880, 0xc0000acba0, 0x661046, 0x491f620, 0xc002b01e90)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc002b01e90, 0xc01558a540, 0xc001681820, 0x5a4e4e0, 0xc002b01e90, 0xc006370600, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007f6c500, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007f6c500, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007f6c500)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218082146 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02aa3eab0, 0x1688c72a2d087206, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02aa3eab0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02aa3ea80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02aa3ea80, 0x4c245c5, 0x10, 0xc02b1dd6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02aa3ea80, 0xc02a466640, 0xc02aa3ea80, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02aa3e9f0, 0xc02a466640, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02aa3e9f0, 0x4bc7440, 0xc02a466640, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02aa3e9f0, 0x4bc7440, 0xc02a466640, 0xc00c71ab20, 0xc00c71ab40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02aa3e9f0, 0x4bc7440, 0xc02a466640, 0xc02b1dd990, 0xca72c8, 0x48dd840, 0xc01f1695c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02aa3e9f0, 0x4bc7440, 0xc02a466640, 0xc00c71ab20, 0xc01f1695c0, 0xc01f1695c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02aa3e9f0, 0x4bc7440, 0xc02a466640, 0x0, 0xc000600000, 0x0, 0xc02a42d800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02aa3e9f0, 0x4bc7440, 0xc02a466640, 0xc00c71ab20, 0xc01f169580, 0xc02b1ddac8, 0xca72c8, 0x48dd840, 0xc01f169580)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02aa3e9f0, 0x4bc7440, 0xc02a466640, 0xc00c71ab20, 0xc00c71ab40, 0xc0013f1ba0, 0x661046, 0x491f620, 0xc02aa3e9f0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02aa3e9f0, 0xc02a42d7a0, 0xc001681820, 0x5a4e4e0, 0xc02aa3e9f0, 0xc02aa63200, 0x2d1)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029480f00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029480f00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029480f00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218080641 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc020fe8e10, 0x1688c728452eb61e, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc020fe8e10, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc020fe8d80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc020fe8d80, 0x4c245c5, 0x10, 0xc0236956c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc020fe8d80, 0xc0169b4e60, 0xc020fe8d80, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc020fe8b40, 0xc0169b4e60, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc020fe8b40, 0x4bc7440, 0xc0169b4e60, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc020fe8b40, 0x4bc7440, 0xc0169b4e60, 0xc0258d9240, 0xc0258d9260, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc020fe8b40, 0x4bc7440, 0xc0169b4e60, 0xc023695990, 0xca72c8, 0x48dd840, 0xc01229ca40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc020fe8b40, 0x4bc7440, 0xc0169b4e60, 0xc0258d9240, 0xc01229ca40, 0xc01229ca40, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc020fe8b40, 0x4bc7440, 0xc0169b4e60, 0x0, 0xc003d76000, 0x0, 0xc00df28960)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc020fe8b40, 0x4bc7440, 0xc0169b4e60, 0xc0258d9240, 0xc01229ca00, 0xc023695ac8, 0xca72c8, 0x48dd840, 0xc01229ca00)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc020fe8b40, 0x4bc7440, 0xc0169b4e60, 0xc0258d9240, 0xc0258d9260, 0xc0000acba0, 0x661046, 0x491f620, 0xc020fe8b40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc020fe8b40, 0xc00df28900, 0xc001681820, 0x5a4e4e0, 0xc020fe8b40, 0xc02351b800, 0x319)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02351c400, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02351c400, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc02351c400)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217817874 [semacquire, 26 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc008b454f8, 0x1688c5cf3ede5ece, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc008b454f8, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01a725110, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01a725110, 0x4c245c5, 0x10, 0xc006ed2180, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01a725110, 0xc009f7d900, 0xc01a725110, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01a725050, 0xc009f7d900, 0xc008b44e78, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc009246000, 0x5a4e4e0, 0xc01a725050, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc009246000, 0x5a6dfa0, 0xc008b44e78, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc008b44e78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc008b44e78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc008b44e78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1145 +0x3e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc008b447f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:784 +0x5e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00142e8d0, 0x5a6dfa0, 0xc008b447f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00142e898, 0x5a6dfa0, 0xc008b447f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00142e868, 0x5a6dfa0, 0xc008b447f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:126 +0x8db
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00142e7c8, 0x5a6dfa0, 0xc008b447f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00142e7c8, 0x5a6dfa0, 0xc008b447f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00142e6f8, 0x5a6dfa0, 0xc008b447f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00142e6d8, 0x5a6dfa0, 0xc008b447f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00142e640, 0x5a6dfa0, 0xc008b447f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00142e480, 0x5a6dfa0, 0xc00885deb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a6dfa0, 0xc00885deb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00a620510, 0x5a6dfa0, 0xc00885deb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcher.func1(0x5a6dfa0, 0xc00885deb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:217 +0x85
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00ed16d20, 0x5a6dfa0, 0xc00885deb8, 0x485e900, 0x401)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:379 +0x739
github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00ed16d20, 0x5a6dfa0, 0xc00885deb8, 0x292dbf7, 0xc01095dcc0, 0x28, 0x28, 0x400, 0x59a93e0, 0xc00d240de0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:480 +0x1ab
github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc00a620870, 0x5a6dfa0, 0xc00885deb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:129 +0x104
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).nextBatch(0xc0053dc1e0, 0x5a6dfa0, 0xc00885deb8, 0xc00b397500, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:803 +0xb9
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:785
github.com/cockroachdb/cockroach/pkg/sql/colmem.(*Allocator).PerformOperation(0xc01a724e40, 0x0, 0x0, 0x0, 0xc006ed74b0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colmem/allocator.go:300 +0x59
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc0053dc1e0, 0x5a6dfa0, 0xc00885deb8, 0xc, 0x5a6dfa0, 0xc00885deb8, 0xc00885dd40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:777 +0xa5
github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc00d3a1200, 0x5a6dfa0, 0xc00885c7f8, 0x2212b73200000000, 0x60c8b0a7)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:109 +0x87
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc01a724ed0, 0x5a6dfa0, 0xc00885c7f8, 0xc006ed75d8, 0xa9fbc6)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:48 +0x69
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc01407ba90, 0x5a6dfa0, 0xc00885c7f8, 0xc006ed0000, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).Next(0xc00521e180, 0x5a6dfa0, 0xc00885c7f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/stats.go:86 +0x5d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc01555ac00, 0xc01407ba90, 0xc006ed76d8, 0x2d229ef)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:290 +0x163
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:315
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc006ed7730, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc01555ac00, 0x5a6dfa0, 0xc00885c7f8, 0x2d229c0, 0x7f8b0dd7deb0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:321 +0x5b
github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run(0x5a6dfa0, 0xc00885c7f8, 0x5a745e0, 0xc01555ac00, 0x5a16ca0, 0xc0243cb880)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:171 +0x35
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc01555ac00, 0x5a4e420, 0xc00b397400)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:781 +0x96
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00ada0900, 0x5a4e420, 0xc00b397400, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc00ed169a0, 0xc00a620510, 0xc0203ce700, 0xc0243cb880, 0xc00ebde6f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc00885c178, 0xc00ebde6f8, 0xc00ed169a0, 0xc00a620510, 0x5a507e0, 0xc009f7c780, 0x0, 0xc0243cb880, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00ebde000, 0x5a6dfa0, 0xc00885c178, 0xc00ebde4e0, 0x3, 0x7f8b0dd7dcb0, 0xc012f38a80, 0xc006ed8000, 0xc010b50dd8, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00ebde000, 0x5a6dfa0, 0xc00885c178, 0xc00ebde4e0, 0x7f8b0dd7dcb0, 0xc012f38a80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00ebde000, 0x5a6dfa0, 0xc00885c178, 0x5a6ce20, 0xc002f280a0, 0xc00b808030, 0xbb, 0x1, 0x1, 0xc013c7c690, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00ebde000, 0x5a4e420, 0xc00b397100, 0x5a6ce20, 0xc002f280a0, 0xc00b808030, 0xbb, 0x1, 0x1, 0xc013c7c690, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00ebde000, 0x5a4e420, 0xc00b397100, 0xc013c7c690, 0xc00ef1ad00, 0x1, 0x1, 0xc015678b40, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc00b808122, 0x0, 0x0, 0x2203b6b7, 0xed85aa7a7, 0x0, 0xc00ebde000, 0xc006ed9a20, 0xc006ed9a10, 0x5, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00ebde000, 0x5a4e420, 0xc00b397100, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00ebde000, 0x5a4e420, 0xc012a216c0, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc00b04ada0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc012a216c0, 0xc00ebde000, 0x5400, 0x15000, 0xc001805a40, 0xc00b04ada0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc014937406, 0xc00fd5bf80, 0x5a4e420, 0xc012a216c0, 0xc00b04ada0, 0xc001a38000, 0xc012f38400, 0x5ab5540, 0xc02116bd00, 0xc01ebce4e0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216086058 [chan receive, 105 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0021c6000, 0x5a4e4e0, 0xc01eb6fd70, 0xc023d312b0, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc01eb6fd70, 0x5ab2120, 0xc010204a80, 0xc0096e8009, 0xd, 0xc01eb6fd10, 0x59fb820, 0xc023010240, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc01eb6fd70, 0x5ab21e0, 0xc016a01260, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc016a01260)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc016a01260, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 218080717 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0141d45a0, 0x1688c7285748301b, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0141d45a0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0141d4570, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0141d4570, 0x4c245c5, 0x10, 0xc023ad36c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0141d4570, 0xc013c89360, 0xc0141d4570, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0141d44b0, 0xc013c89360, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0141d44b0, 0x4bc7440, 0xc013c89360, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0141d44b0, 0x4bc7440, 0xc013c89360, 0xc0044284e0, 0xc004428520, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0141d44b0, 0x4bc7440, 0xc013c89360, 0xc023ad3990, 0xca72c8, 0x48dd840, 0xc021884380)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0141d44b0, 0x4bc7440, 0xc013c89360, 0xc0044284e0, 0xc021884380, 0xc021884380, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0141d44b0, 0x4bc7440, 0xc013c89360, 0x0, 0x87067e0, 0x0, 0xc02441b020)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0141d44b0, 0x4bc7440, 0xc013c89360, 0xc0044284e0, 0xc021884340, 0xc023ad3ac8, 0xca72c8, 0x48dd840, 0xc021884340)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0141d44b0, 0x4bc7440, 0xc013c89360, 0xc0044284e0, 0xc004428520, 0xc0072a0ba0, 0x661046, 0x491f620, 0xc0141d44b0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0141d44b0, 0xc02441afc0, 0xc001681820, 0x5a4e4e0, 0xc0141d44b0, 0xc023a8ec00, 0x2f0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc023a80500, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc023a80500, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc023a80500)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216086026 [chan receive, 105 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc01e3aa400, 0x5a4e4e0, 0xc01eb6e1e0, 0xc023d30d30, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc01eb6e1e0, 0x5ab2120, 0xc015c7a000, 0xc009697028, 0xd, 0xc01eb6e1b0, 0x59fb820, 0xc00c600ed0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc01eb6e1e0, 0x5ab21e0, 0xc0128979e0, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc0128979e0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc0128979e0, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 218080111 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0092d9170, 0x1688c7281e62a340, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0092d9170, 0x1688c7281e62a340, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0092d9110, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0092d9110, 0x4c245c5, 0x10, 0xc0216876c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0092d9110, 0xc016c17c20, 0xc0092d9110, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0092d90b0, 0xc016c17c20, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0092d90b0, 0x4bc7440, 0xc016c17c20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0092d90b0, 0x4bc7440, 0xc016c17c20, 0xc00a6a33a0, 0xc00a6a33c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0092d90b0, 0x4bc7440, 0xc016c17c20, 0xc021687990, 0xca72c8, 0x48dd840, 0xc005727dc0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0092d90b0, 0x4bc7440, 0xc016c17c20, 0xc00a6a33a0, 0xc005727dc0, 0xc005727dc0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0092d90b0, 0x4bc7440, 0xc016c17c20, 0x0, 0xc00758a000, 0x0, 0xc029fa7440)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0092d90b0, 0x4bc7440, 0xc016c17c20, 0xc00a6a33a0, 0xc005727d80, 0xc021687ac8, 0xca72c8, 0x48dd840, 0xc005727d80)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0092d90b0, 0x4bc7440, 0xc016c17c20, 0xc00a6a33a0, 0xc00a6a33c0, 0xc0013f1ba0, 0x661046, 0x491f620, 0xc0092d90b0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0092d90b0, 0xc029fa73e0, 0xc001681820, 0x5a4e4e0, 0xc0092d90b0, 0xc020ec9000, 0x41d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00c477a00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00c477a00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00c477a00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218080301 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0195ffaa0, 0x1688c72823a1c417, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0195ffaa0, 0x1688c72823a1c417, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0195ffa70, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0195ffa70, 0x4c245c5, 0x10, 0xc021f356c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0195ffa70, 0xc0297aa780, 0xc0195ffa70, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0195ffa10, 0xc0297aa780, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0195ffa10, 0x4bc7440, 0xc0297aa780, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0195ffa10, 0x4bc7440, 0xc0297aa780, 0xc020b4f720, 0xc020b4f740, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0195ffa10, 0x4bc7440, 0xc0297aa780, 0xc021f35990, 0xca72c8, 0x48dd840, 0xc00f8fab00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0195ffa10, 0x4bc7440, 0xc0297aa780, 0xc020b4f720, 0xc00f8fab00, 0xc00f8fab00, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0195ffa10, 0x4bc7440, 0xc0297aa780, 0x0, 0xc00fba5800, 0x0, 0xc0296d3b00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0195ffa10, 0x4bc7440, 0xc0297aa780, 0xc020b4f720, 0xc00f8faac0, 0xc021f35ac8, 0xca72c8, 0x48dd840, 0xc00f8faac0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0195ffa10, 0x4bc7440, 0xc0297aa780, 0xc020b4f720, 0xc020b4f740, 0xc003a32ba0, 0x661046, 0x491f620, 0xc0195ffa10)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0195ffa10, 0xc0296d3aa0, 0xc001681820, 0x5a4e4e0, 0xc0195ffa10, 0xc021a31000, 0x429)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc021563300, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc021563300, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc021563300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216090000 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc009762000, 0x1688bcc2310517b2, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc009762000, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc014d4bfb0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc014d4bfb0, 0x4c245c5, 0x10, 0xc016d696c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc014d4bfb0, 0xc0144a63c0, 0xc014d4bfb0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc014d4bf20, 0xc0144a63c0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc014d4bf20, 0x4bc7440, 0xc0144a63c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc014d4bf20, 0x4bc7440, 0xc0144a63c0, 0xc020a8f8a0, 0xc020a8f8c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc014d4bf20, 0x4bc7440, 0xc0144a63c0, 0xc016d69990, 0xca72c8, 0x48dd840, 0xc022068380)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc014d4bf20, 0x4bc7440, 0xc0144a63c0, 0xc020a8f8a0, 0xc022068380, 0xc022068380, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc014d4bf20, 0x4bc7440, 0xc0144a63c0, 0x0, 0xc00009c800, 0x0, 0xc00bda7500)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc014d4bf20, 0x4bc7440, 0xc0144a63c0, 0xc020a8f8a0, 0xc022068300, 0xc016d69ac8, 0xca72c8, 0x48dd840, 0xc022068300)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc014d4bf20, 0x4bc7440, 0xc0144a63c0, 0xc020a8f8a0, 0xc020a8f8c0, 0xc0013f4ba0, 0x661046, 0x491f620, 0xc014d4bf20)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc014d4bf20, 0xc00bda74a0, 0xc001681820, 0x5a4e4e0, 0xc014d4bf20, 0xc014498c00, 0x32b)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc012ac5100, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc012ac5100, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc012ac5100)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218080742 [chan receive]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0232fe400, 0x5a4e4e0, 0xc01a7fe000, 0xc00aae9740, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc01a7fe000, 0x5ab2120, 0xc02390a700, 0xc023957028, 0xd, 0xc02b9e5fb0, 0x59fb820, 0xc02b9e5a70, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc01a7fe000, 0x5ab21e0, 0xc0256982a0, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc0256982a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc0256982a0, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 218080569 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01b2408d0, 0x1688c72837d3a64f, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01b2408d0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01b240870, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01b240870, 0x4c245c5, 0x10, 0xc0233096c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01b240870, 0xc0296be1e0, 0xc01b240870, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01b2407b0, 0xc0296be1e0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01b2407b0, 0x4bc7440, 0xc0296be1e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01b2407b0, 0x4bc7440, 0xc0296be1e0, 0xc016bdc0c0, 0xc016bdc0e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01b2407b0, 0x4bc7440, 0xc0296be1e0, 0xc023309990, 0xca72c8, 0x48dd840, 0xc01fb64c00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01b2407b0, 0x4bc7440, 0xc0296be1e0, 0xc016bdc0c0, 0xc01fb64c00, 0xc01fb64c00, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01b2407b0, 0x4bc7440, 0xc0296be1e0, 0x0, 0xc003d76000, 0x0, 0xc00b398a20)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01b2407b0, 0x4bc7440, 0xc0296be1e0, 0xc016bdc0c0, 0xc01fb64bc0, 0xc023309ac8, 0xca72c8, 0x48dd840, 0xc01fb64bc0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01b2407b0, 0x4bc7440, 0xc0296be1e0, 0xc016bdc0c0, 0xc016bdc0e0, 0xc0000adba0, 0x661046, 0x491f620, 0xc01b2407b0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01b2407b0, 0xc00b3989c0, 0xc001681820, 0x5a4e4e0, 0xc01b2407b0, 0xc0231eec00, 0x2df)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc022b97600, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc022b97600, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc022b97600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218071626 [semacquire, 3 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01b2c8810, 0x1688c71c59440591, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01b2c8810, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01b2c87e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01b2c87e0, 0x4c245c5, 0x10, 0xc01e1f96c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01b2c87e0, 0xc02b2e6dc0, 0xc01b2c87e0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01b2c8780, 0xc02b2e6dc0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01b2c8780, 0x4bc7440, 0xc02b2e6dc0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01b2c8780, 0x4bc7440, 0xc02b2e6dc0, 0xc01e730ec0, 0xc01e730ee0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01b2c8780, 0x4bc7440, 0xc02b2e6dc0, 0xc01e1f9990, 0xca72c8, 0x48dd840, 0xc011c24540)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01b2c8780, 0x4bc7440, 0xc02b2e6dc0, 0xc01e730ec0, 0xc011c24540, 0xc011c24540, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01b2c8780, 0x4bc7440, 0xc02b2e6dc0, 0x0, 0xc00e8e4800, 0x0, 0xc0154ee480)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01b2c8780, 0x4bc7440, 0xc02b2e6dc0, 0xc01e730ec0, 0xc011c24500, 0xc01e1f9ac8, 0xca72c8, 0x48dd840, 0xc011c24500)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01b2c8780, 0x4bc7440, 0xc02b2e6dc0, 0xc01e730ec0, 0xc01e730ee0, 0xc003016ba0, 0x661046, 0x491f620, 0xc01b2c8780)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01b2c8780, 0xc0154ee420, 0xc001681820, 0x5a4e4e0, 0xc01b2c8780, 0xc01e062000, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01df60b00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01df60b00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc01df60b00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216997624 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc00542cfc0, 0x1688c18c61c7970f, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc00542cfc0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00542cf90, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00542cf90, 0x4c245c5, 0x10, 0xc004deb6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00542cf90, 0xc01dff08c0, 0xc00542cf90, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00542cf30, 0xc01dff08c0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc00542cf30, 0x4bc7440, 0xc01dff08c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc00542cf30, 0x4bc7440, 0xc01dff08c0, 0xc01d79e500, 0xc01d79e520, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00542cf30, 0x4bc7440, 0xc01dff08c0, 0xc004deb990, 0xca72c8, 0x48dd840, 0xc0100b54c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc00542cf30, 0x4bc7440, 0xc01dff08c0, 0xc01d79e500, 0xc0100b54c0, 0xc0100b54c0, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00542cf30, 0x4bc7440, 0xc01dff08c0, 0x0, 0x87067e0, 0x0, 0xc01325bbc0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc00542cf30, 0x4bc7440, 0xc01dff08c0, 0xc01d79e500, 0xc0100b5440, 0xc004debac8, 0xca72c8, 0x48dd840, 0xc0100b5440)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc00542cf30, 0x4bc7440, 0xc01dff08c0, 0xc01d79e500, 0xc01d79e520, 0xc02eebaba0, 0x661046, 0x491f620, 0xc00542cf30)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc00542cf30, 0xc01325bb00, 0xc001681820, 0x5a4e4e0, 0xc00542cf30, 0xc00e699800, 0x344)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00ada4800, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00ada4800, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00ada4800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216996892 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0227dae70, 0x1688c18bc11afbc8, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0227dae70, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0227dae40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0227dae40, 0x4c245c5, 0x10, 0xc0042876c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0227dae40, 0xc01aeb8e60, 0xc0227dae40, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0227dad20, 0xc01aeb8e60, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0227dad20, 0x4bc7440, 0xc01aeb8e60, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0227dad20, 0x4bc7440, 0xc01aeb8e60, 0xc00b33fec0, 0xc00b33fee0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0227dad20, 0x4bc7440, 0xc01aeb8e60, 0xc004287990, 0xca72c8, 0x48dd840, 0xc00df807c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0227dad20, 0x4bc7440, 0xc01aeb8e60, 0xc00b33fec0, 0xc00df807c0, 0xc00df807c0, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0227dad20, 0x4bc7440, 0xc01aeb8e60, 0x0, 0xc00fba5800, 0x0, 0xc014527b60)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0227dad20, 0x4bc7440, 0xc01aeb8e60, 0xc00b33fec0, 0xc00df80780, 0xc004287ac8, 0xca72c8, 0x48dd840, 0xc00df80780)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0227dad20, 0x4bc7440, 0xc01aeb8e60, 0xc00b33fec0, 0xc00b33fee0, 0xc003a33ba0, 0x661046, 0x491f620, 0xc0227dad20)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0227dad20, 0xc014527b00, 0xc001681820, 0x5a4e4e0, 0xc0227dad20, 0xc01f9acc00, 0x341)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00b66c600, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00b66c600, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00b66c600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217748449 [semacquire, 32 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01d64b0e0, 0x1688c57d7ecb2a6a, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01d64b0e0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01d64b0e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01d64b0e0, 0x4c245c5, 0x10, 0xc00538b6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01d64b0e0, 0xc00aad3e00, 0xc01d64b0e0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a6dfa0, 0xc002270b38, 0xc00aad3e00, 0xc0058cac00, 0xc01d64b080, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a6dfa0, 0xc002270b38, 0x4bc7440, 0xc00aad3e00, 0x21, 0xc00538b890, 0x2, 0x2)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01d64af90, 0x4bc7440, 0xc00aad3e00, 0xc00fbc7340, 0xc00fbc7360, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:137 +0x2dc
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01d64af90, 0x4bc7440, 0xc00aad3e00, 0xc00538b990, 0xca72c8, 0x48dd840, 0xc0149a12c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01d64af90, 0x4bc7440, 0xc00aad3e00, 0xc00fbc7340, 0xc0149a12c0, 0xc0149a12c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01d64af90, 0x4bc7440, 0xc00aad3e00, 0x0, 0xc0002a9800, 0x0, 0xc028e33560)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01d64af90, 0x4bc7440, 0xc00aad3e00, 0xc00fbc7340, 0xc0149a1280, 0xc00538bac8, 0xca72c8, 0x48dd840, 0xc0149a1280)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01d64af90, 0x4bc7440, 0xc00aad3e00, 0xc00fbc7340, 0xc00fbc7360, 0xc002f74ba0, 0x661046, 0x491f620, 0xc01d64af90)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01d64af90, 0xc028e33500, 0xc001681820, 0x5a4e4e0, 0xc01d64af90, 0xc003cae000, 0x343)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00d04b100, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00d04b100, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00d04b100)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218080015 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc0202ff838, 0x1688c7281b2836a9, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc0202ff838, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00bd7ab70, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00bd7ab70, 0x4c245c5, 0x10, 0xc02583a3f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00bd7ab70, 0xc01ae972c0, 0xc00bd7ab70, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00bd7ab40, 0xc01ae972c0, 0xc0202ff4f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00c61a5c0, 0x5a4e4e0, 0xc00bd7ab40, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00c61a5c0, 0x5a6dfa0, 0xc0202ff4f8, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc0202ff4f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc0202ff4f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc0202ff4f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc0202ff4f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc0202ff358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0059abf50, 0x5a6dfa0, 0xc0202ff358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc0059abf18, 0x5a6dfa0, 0xc0202ff358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc0059abee8, 0x5a6dfa0, 0xc0202ff358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc0059abe48, 0x5a6dfa0, 0xc0202ff358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc0059abe48, 0x5a6dfa0, 0xc0202ff358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc0059abd78, 0x5a6dfa0, 0xc0202ff358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc0059abd58, 0x5a6dfa0, 0xc0202ff358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0059abcc0, 0x5a6dfa0, 0xc0202ff358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc0059abb00, 0x5a4e420, 0xc010f70380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc010f70380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc02086dcb0, 0x5a4e420, 0xc010f70380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc010f70380, 0xc02583f1f0, 0xc012208f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc02086dcb0, 0x5a4e420, 0xc010f70380, 0xc012208f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc02086dcb0, 0x5a4e420, 0xc010f70380, 0xc012208f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc020dfa628, 0x5a4e420, 0xc010f70380, 0xc0058cfb00, 0xc024177de0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc020dfa600, 0x5a4e420, 0xc010f70380, 0xc0058cfb00, 0xc024177de0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc027744500, 0x5a4e420, 0xc010f70380, 0xc0058cfb00, 0xc024177de0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc027744500, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc02583f588, 0x4bfc8f0, 0x5, 0x5a50860, 0xc027744500)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc02583f588, 0x5a50860, 0xc027744500, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc02583f588, 0x5a50860, 0xc027744500, 0xc, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc010f70380, 0xc0058cfb00, 0xc024177de0, 0x5a50860, 0xc027744500, 0x5a4e420, 0xc010f70380)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc020dfac00, 0x5a4e420, 0xc010f70380)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc01319a900)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc02583f768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc020315800, 0x5a4e420, 0xc010f70380)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc020315800, 0x5a4e420, 0xc010f70380)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc021eaac60, 0x5a4e420, 0xc010f70380, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc008cfa380, 0xc02086dcb0, 0xc013ee1b00, 0xc02899d180, 0xc024177ff8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc0203b54f8, 0xc024177ff8, 0xc008cfa380, 0xc02086dcb0, 0x5a50860, 0xc027744500, 0x0, 0xc02899d180, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc024177900, 0x5a6dfa0, 0xc0203b54f8, 0xc024177de0, 0x2, 0x7f8b0dd7dcb0, 0xc009e55880, 0xc025840000, 0xc029a6e4d8, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc024177900, 0x5a6dfa0, 0xc0203b54f8, 0xc024177de0, 0x7f8b0dd7dcb0, 0xc009e55880, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc024177900, 0x5a6dfa0, 0xc0203b54f8, 0x5a6dda0, 0xc013ee1300, 0xc020094d63, 0x1e9, 0x11, 0x1, 0xc0203c5770, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc024177900, 0x5a4e420, 0xc011be7780, 0x5a6dda0, 0xc013ee1300, 0xc020094d63, 0x1e9, 0x11, 0x1, 0xc0203c5770, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc024177900, 0x5a4e420, 0xc011be7780, 0xc0203c5770, 0xc021eaa480, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc020f0c171, 0x0, 0x0, 0x2bd3494a, 0xed85aad70, 0x0, 0xc024177900, 0xc025841a20, 0xc025841a10, 0x2d2, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc024177900, 0x5a4e420, 0xc011be7780, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc024177900, 0x5a4e420, 0xc01418b840, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc0034c8c60, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc01418b840, 0xc024177900, 0x5400, 0x15000, 0xc001805a40, 0xc0034c8c60, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc012af61e1, 0xc01e353500, 0x5a4e420, 0xc01418b840, 0xc0034c8c60, 0xc001a38000, 0xc009e55200, 0x5ab5540, 0xc029ae6400, 0xc0070ea180, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216798469 [semacquire, 132 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01f698570, 0x1688c00793416649, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01f698570, 0x1688c00793416649, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01f698510, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01f698510, 0x4c245c5, 0x10, 0xc0048336c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01f698510, 0xc013029f40, 0xc01f698510, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01f6984b0, 0xc013029f40, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01f6984b0, 0x4bc7440, 0xc013029f40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01f6984b0, 0x4bc7440, 0xc013029f40, 0xc02197af80, 0xc02197afa0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01f6984b0, 0x4bc7440, 0xc013029f40, 0xc004833990, 0xca72c8, 0x48dd840, 0xc004fed4c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01f6984b0, 0x4bc7440, 0xc013029f40, 0xc02197af80, 0xc004fed4c0, 0xc004fed4c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01f6984b0, 0x4bc7440, 0xc013029f40, 0x0, 0xc024c79800, 0x0, 0xc01d87db60)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01f6984b0, 0x4bc7440, 0xc013029f40, 0xc02197af80, 0xc004fed480, 0xc004833ac8, 0xca72c8, 0x48dd840, 0xc004fed480)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01f6984b0, 0x4bc7440, 0xc013029f40, 0xc02197af80, 0xc02197afa0, 0xc0037e7ba0, 0x661046, 0x491f620, 0xc01f6984b0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01f6984b0, 0xc01d87db00, 0xc001681820, 0x5a4e4e0, 0xc01f6984b0, 0xc00ac02000, 0x431)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f559e00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f559e00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f559e00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217179966 [semacquire, 88 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc00dfa2960, 0x1688c270bda65485, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc00dfa2960, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00dfa2930, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00dfa2930, 0x4c245c5, 0x10, 0xc0048f16c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00dfa2930, 0xc0191c3040, 0xc00dfa2930, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00dfa28d0, 0xc0191c3040, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc00dfa28d0, 0x4bc7440, 0xc0191c3040, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc00dfa28d0, 0x4bc7440, 0xc0191c3040, 0xc01ae28a80, 0xc01ae28aa0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00dfa28d0, 0x4bc7440, 0xc0191c3040, 0xc0048f1990, 0xca72c8, 0x48dd840, 0xc00d859d40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc00dfa28d0, 0x4bc7440, 0xc0191c3040, 0xc01ae28a80, 0xc00d859d40, 0xc00d859d40, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00dfa28d0, 0x4bc7440, 0xc0191c3040, 0x0, 0xc00fba5800, 0x0, 0xc027560ba0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc00dfa28d0, 0x4bc7440, 0xc0191c3040, 0xc01ae28a80, 0xc00d859d00, 0xc0048f1ac8, 0xca72c8, 0x48dd840, 0xc00d859d00)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc00dfa28d0, 0x4bc7440, 0xc0191c3040, 0xc01ae28a80, 0xc01ae28aa0, 0xc002294ba0, 0x661046, 0x491f620, 0xc00dfa28d0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc00dfa28d0, 0xc027560b40, 0xc001681820, 0x5a4e4e0, 0xc00dfa28d0, 0xc00ec7a600, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007f6c200, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007f6c200, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007f6c200)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216593286 [semacquire, 150 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0142d6cc0, 0x1688bf146876f200, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0142d6cc0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0142d6c90, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0142d6c90, 0x4c245c5, 0x10, 0xc00789b6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0142d6c90, 0xc019a68be0, 0xc0142d6c90, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0142d6b40, 0xc019a68be0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0142d6b40, 0x4bc7440, 0xc019a68be0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0142d6b40, 0x4bc7440, 0xc019a68be0, 0xc019bf85a0, 0xc019bf85c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0142d6b40, 0x4bc7440, 0xc019a68be0, 0xc00789b990, 0xca72c8, 0x48dd840, 0xc012f3b740)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0142d6b40, 0x4bc7440, 0xc019a68be0, 0xc019bf85a0, 0xc012f3b740, 0xc012f3b740, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0142d6b40, 0x4bc7440, 0xc019a68be0, 0x0, 0xc003d76000, 0x0, 0xc012369e00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0142d6b40, 0x4bc7440, 0xc019a68be0, 0xc019bf85a0, 0xc012f3b700, 0xc00789bac8, 0xca72c8, 0x48dd840, 0xc012f3b700)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0142d6b40, 0x4bc7440, 0xc019a68be0, 0xc019bf85a0, 0xc019bf85c0, 0xc00190fba0, 0x661046, 0x491f620, 0xc0142d6b40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0142d6b40, 0xc012369da0, 0xc001681820, 0x5a4e4e0, 0xc0142d6b40, 0xc004c72600, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc014c3b300, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc014c3b300, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc014c3b300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216616958 [semacquire, 148 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc006360b40, 0x1688bf31835df64d, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc006360b40, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc006360b10, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc006360b10, 0x4c245c5, 0x10, 0xc00240b6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc006360b10, 0xc009e300a0, 0xc006360b10, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc006360ab0, 0xc009e300a0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc006360ab0, 0x4bc7440, 0xc009e300a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc006360ab0, 0x4bc7440, 0xc009e300a0, 0xc00b7202c0, 0xc00b7202e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc006360ab0, 0x4bc7440, 0xc009e300a0, 0xc00240b990, 0xca72c8, 0x48dd840, 0xc0132a2240)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc006360ab0, 0x4bc7440, 0xc009e300a0, 0xc00b7202c0, 0xc0132a2240, 0xc0132a2240, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc006360ab0, 0x4bc7440, 0xc009e300a0, 0x0, 0xc003d76000, 0x0, 0xc00daaa120)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc006360ab0, 0x4bc7440, 0xc009e300a0, 0xc00b7202c0, 0xc0132a2200, 0xc00240bac8, 0xca72c8, 0x48dd840, 0xc0132a2200)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc006360ab0, 0x4bc7440, 0xc009e300a0, 0xc00b7202c0, 0xc00b7202e0, 0xc0072a0ba0, 0x661046, 0x491f620, 0xc006360ab0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc006360ab0, 0xc00daaa060, 0xc001681820, 0x5a4e4e0, 0xc006360ab0, 0xc010aac600, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00b96a300, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00b96a300, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00b96a300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217739623 [semacquire, 33 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0163d8840, 0x1688c56ec4a36ff4, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0163d8840, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0163d8810, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0163d8810, 0x4c245c5, 0x10, 0xc0010ef6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0163d8810, 0xc00f716dc0, 0xc0163d8810, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0163d87b0, 0xc00f716dc0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0163d87b0, 0x4bc7440, 0xc00f716dc0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0163d87b0, 0x4bc7440, 0xc00f716dc0, 0xc0111c8340, 0xc0111c8360, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0163d87b0, 0x4bc7440, 0xc00f716dc0, 0xc0010ef990, 0xca72c8, 0x48dd840, 0xc014bf2100)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0163d87b0, 0x4bc7440, 0xc00f716dc0, 0xc0111c8340, 0xc014bf2100, 0xc014bf2100, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0163d87b0, 0x4bc7440, 0xc00f716dc0, 0x0, 0x87067e0, 0x0, 0xc022c4c8a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0163d87b0, 0x4bc7440, 0xc00f716dc0, 0xc0111c8340, 0xc014bf20c0, 0xc0010efac8, 0xca72c8, 0x48dd840, 0xc014bf20c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0163d87b0, 0x4bc7440, 0xc00f716dc0, 0xc0111c8340, 0xc0111c8360, 0xc0072a2ba0, 0x661046, 0x491f620, 0xc0163d87b0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0163d87b0, 0xc022c4c7e0, 0xc001681820, 0x5a4e4e0, 0xc0163d87b0, 0xc00bc1d200, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00efa8000, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00efa8000, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00efa8000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218082003 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02976b470, 0x1688c72a276db597, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02976b470, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02976b440, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02976b440, 0x4c245c5, 0x10, 0xc02a20d6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02976b440, 0xc029766500, 0xc02976b440, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02976b3e0, 0xc029766500, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02976b3e0, 0x4bc7440, 0xc029766500, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02976b3e0, 0x4bc7440, 0xc029766500, 0xc01954b440, 0xc01954b460, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02976b3e0, 0x4bc7440, 0xc029766500, 0xc02a20d990, 0xca72c8, 0x48dd840, 0xc01527cb00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02976b3e0, 0x4bc7440, 0xc029766500, 0xc01954b440, 0xc01527cb00, 0xc01527cb00, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02976b3e0, 0x4bc7440, 0xc029766500, 0x0, 0xc000100800, 0x0, 0xc029723e60)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02976b3e0, 0x4bc7440, 0xc029766500, 0xc01954b440, 0xc01527cac0, 0xc02a20dac8, 0xca72c8, 0x48dd840, 0xc01527cac0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02976b3e0, 0x4bc7440, 0xc029766500, 0xc01954b440, 0xc01954b460, 0xc0013eeba0, 0x661046, 0x491f620, 0xc02976b3e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02976b3e0, 0xc029723e00, 0xc001681820, 0x5a4e4e0, 0xc02976b3e0, 0xc029f9e000, 0x167e)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029a4d400, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029a4d400, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc029a4d400)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216997642 [chan receive, 31 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0142bb600, 0x5a4e4e0, 0xc020d1eba0, 0xc0072f6170, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc020d1eba0, 0x5ab2120, 0xc0136bc700, 0xc008283028, 0xd, 0xc020d1eb40, 0x59fb820, 0xc016d10a80, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc020d1eba0, 0x5ab21e0, 0xc01101bec0, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc01101bec0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc01101bec0, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 216801620 [semacquire, 132 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc00f5bd470, 0x1688c00c495f6088, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc00f5bd470, 0x1688c00c495f6088, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00f5bd440, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00f5bd440, 0x4c245c5, 0x10, 0xc01e9c76c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00f5bd440, 0xc00294f2c0, 0xc00f5bd440, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00f5bd3e0, 0xc00294f2c0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc00f5bd3e0, 0x4bc7440, 0xc00294f2c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc00f5bd3e0, 0x4bc7440, 0xc00294f2c0, 0xc00b0c83a0, 0xc00b0c83c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00f5bd3e0, 0x4bc7440, 0xc00294f2c0, 0xc01e9c7990, 0xca72c8, 0x48dd840, 0xc011353340)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc00f5bd3e0, 0x4bc7440, 0xc00294f2c0, 0xc00b0c83a0, 0xc011353340, 0xc011353340, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00f5bd3e0, 0x4bc7440, 0xc00294f2c0, 0x0, 0xc003d76000, 0x0, 0xc01d7aa480)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc00f5bd3e0, 0x4bc7440, 0xc00294f2c0, 0xc00b0c83a0, 0xc0113531c0, 0xc01e9c7ac8, 0xca72c8, 0x48dd840, 0xc0113531c0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc00f5bd3e0, 0x4bc7440, 0xc00294f2c0, 0xc00b0c83a0, 0xc00b0c83c0, 0xc004b46ba0, 0x661046, 0x491f620, 0xc00f5bd3e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc00f5bd3e0, 0xc01d7aa420, 0xc001681820, 0x5a4e4e0, 0xc00f5bd3e0, 0xc013645000, 0x429)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f1b4600, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f1b4600, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00f1b4600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217169836 [semacquire, 89 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc022fee270, 0x1688c262c55a63b8, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc022fee270, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc022fee240, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc022fee240, 0x4c245c5, 0x10, 0xc005d016c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc022fee240, 0xc01fc66be0, 0xc022fee240, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc022fee1e0, 0xc01fc66be0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc022fee1e0, 0x4bc7440, 0xc01fc66be0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc022fee1e0, 0x4bc7440, 0xc01fc66be0, 0xc009e50100, 0xc009e50120, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc022fee1e0, 0x4bc7440, 0xc01fc66be0, 0xc005d01990, 0xca72c8, 0x48dd840, 0xc01f2d6e40)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc022fee1e0, 0x4bc7440, 0xc01fc66be0, 0xc009e50100, 0xc01f2d6e40, 0xc01f2d6e40, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc022fee1e0, 0x4bc7440, 0xc01fc66be0, 0x0, 0xc00fba5800, 0x0, 0xc0035fa9c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc022fee1e0, 0x4bc7440, 0xc01fc66be0, 0xc009e50100, 0xc01f2d6e00, 0xc005d01ac8, 0xca72c8, 0x48dd840, 0xc01f2d6e00)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc022fee1e0, 0x4bc7440, 0xc01fc66be0, 0xc009e50100, 0xc009e50120, 0xc004b46ba0, 0x661046, 0x491f620, 0xc022fee1e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc022fee1e0, 0xc0035fa960, 0xc001681820, 0x5a4e4e0, 0xc022fee1e0, 0xc015e5f200, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00eaa3b00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00eaa3b00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00eaa3b00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217778296 [semacquire, 30 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc00608beb8, 0x1688c5a37fd71ece, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc00608beb8, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0199db830, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0199db830, 0x4c245c5, 0x10, 0xc02e0ce3f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0199db830, 0xc01371f680, 0xc0199db830, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0199db800, 0xc01371f680, 0xc00608bb78, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc010814140, 0x5a4e4e0, 0xc0199db800, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc010814140, 0x5a6dfa0, 0xc00608bb78, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc00608bb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc00608bb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc00608bb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc00608bb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc00608b698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc003530450, 0x5a6dfa0, 0xc00608b698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc003530418, 0x5a6dfa0, 0xc00608b698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc0035303e8, 0x5a6dfa0, 0xc00608b698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc003530348, 0x5a6dfa0, 0xc00608b698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc003530348, 0x5a6dfa0, 0xc00608b698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc003530278, 0x5a6dfa0, 0xc00608b698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc003530258, 0x5a6dfa0, 0xc00608b698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0035301c0, 0x5a6dfa0, 0xc00608b698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc003530000, 0x5a4e420, 0xc00e5b9a40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc00e5b9a40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc01a3d2900, 0x5a4e420, 0xc00e5b9a40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc00e5b9a40, 0xc02e0d31f0, 0xc012b00f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc01a3d2900, 0x5a4e420, 0xc00e5b9a40, 0xc012b00f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc01a3d2900, 0x5a4e420, 0xc00e5b9a40, 0xc012b00f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc022f79828, 0x5a4e420, 0xc00e5b9a40, 0xc00f8de000, 0xc012124ae0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc022f79800, 0x5a4e420, 0xc00e5b9a40, 0xc00f8de000, 0xc012124ae0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc008a963c0, 0x5a4e420, 0xc00e5b9a40, 0xc00f8de000, 0xc012124ae0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc008a963c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc02e0d3588, 0x4bfc8f0, 0x5, 0x5a50860, 0xc008a963c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc014667588, 0x5a50860, 0xc008a963c0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc014667588, 0x5a50860, 0xc008a963c0, 0xc, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc00e5b9a40, 0xc00f8de000, 0xc012124ae0, 0x5a50860, 0xc008a963c0, 0x5a4e420, 0xc00e5b9a40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc01f2bb800, 0x5a4e420, 0xc00e5b9a40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc00257c900)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc014667768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc00f8b4000, 0x5a4e420, 0xc00e5b9a40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc00f8b4000, 0x5a4e420, 0xc00e5b9a40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00c222120, 0x5a4e420, 0xc00e5b9a40, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc0082bd6c0, 0xc01a3d2900, 0xc003203500, 0xc01199bc00, 0xc012124cf8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc005228b38, 0xc012124cf8, 0xc0082bd6c0, 0xc01a3d2900, 0x5a50860, 0xc008a963c0, 0x0, 0xc01199bc00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc012124600, 0x5a6dfa0, 0xc005228b38, 0xc012124ae0, 0x2, 0x7f8b0dd7dcb0, 0xc0243c2680, 0xc014668000, 0xc01b45c178, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc012124600, 0x5a6dfa0, 0xc005228b38, 0xc012124ae0, 0x7f8b0dd7dcb0, 0xc0243c2680, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc012124600, 0x5a6dfa0, 0xc005228b38, 0x5a6dda0, 0xc01d4cf300, 0xc00c292030, 0x1e9, 0x11, 0x1, 0xc014aeeff0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc012124600, 0x5a4e420, 0xc0244d45c0, 0x5a6dda0, 0xc01d4cf300, 0xc00c292030, 0x1e9, 0x11, 0x1, 0xc014aeeff0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc012124600, 0x5a4e420, 0xc0244d45c0, 0xc014aeeff0, 0xc008c7b7a0, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc00c292393, 0x0, 0x0, 0x28a8b69f, 0xed85aa6eb, 0x0, 0xc012124600, 0xc02e0d5a20, 0xc02e0d5a10, 0x5, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc012124600, 0x5a4e420, 0xc0244d45c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc012124600, 0x5a4e420, 0xc00e7b2740, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc01b0fe330, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc00e7b2740, 0xc012124600, 0x5400, 0x15000, 0xc001805a40, 0xc01b0fe330, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc013490f41, 0xc01aa6c200, 0x5a4e420, 0xc00e7b2740, 0xc01b0fe330, 0xc001a38000, 0xc0243c2000, 0x5ab5540, 0xc0119e9080, 0xc016e50b40, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216096157 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc00c62f770, 0x1688bcc529fce53a, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc00c62f770, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00c62f710, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00c62f710, 0x4c245c5, 0x10, 0xc0031b96c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00c62f710, 0xc01dff1720, 0xc00c62f710, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00c62f6b0, 0xc01dff1720, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc00c62f6b0, 0x4bc7440, 0xc01dff1720, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc00c62f6b0, 0x4bc7440, 0xc01dff1720, 0xc00f5c1760, 0xc00f5c1780, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00c62f6b0, 0x4bc7440, 0xc01dff1720, 0xc0031b9990, 0xca72c8, 0x48dd840, 0xc00eafe240)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc00c62f6b0, 0x4bc7440, 0xc01dff1720, 0xc00f5c1760, 0xc00eafe240, 0xc00eafe240, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00c62f6b0, 0x4bc7440, 0xc01dff1720, 0x0, 0xc00758a000, 0x0, 0xc01ae331a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc00c62f6b0, 0x4bc7440, 0xc01dff1720, 0xc00f5c1760, 0xc00eafe200, 0xc0031b9ac8, 0xca72c8, 0x48dd840, 0xc00eafe200)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc00c62f6b0, 0x4bc7440, 0xc01dff1720, 0xc00f5c1760, 0xc00f5c1780, 0xc0000adba0, 0x661046, 0x491f620, 0xc00c62f6b0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc00c62f6b0, 0xc01ae33140, 0xc001681820, 0x5a4e4e0, 0xc00c62f6b0, 0xc00311b800, 0x33a)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc007cf5400, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc007cf5400, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc007cf5400)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217748646 [semacquire, 32 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc01301b9d8, 0x1688c57d40789e98, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc01301b9d8, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01b48e960, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01b48e960, 0x4c245c5, 0x10, 0xc02a3f83f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01b48e960, 0xc019518b40, 0xc01b48e960, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01b48e930, 0xc019518b40, 0xc01301b1b8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc01b0f4700, 0x5a4e4e0, 0xc01b48e930, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc01b0f4700, 0x5a6dfa0, 0xc01301b1b8, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc01301b1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc01301b1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc01301b1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc01301b1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc007853eb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc006d60d50, 0x5a6dfa0, 0xc007853eb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc006d60d18, 0x5a6dfa0, 0xc007853eb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc006d60ce8, 0x5a6dfa0, 0xc007853eb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc006d60c48, 0x5a6dfa0, 0xc007853eb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc006d60c48, 0x5a6dfa0, 0xc007853eb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc006d60b78, 0x5a6dfa0, 0xc007853eb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc006d60b58, 0x5a6dfa0, 0xc007853eb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc006d60ac0, 0x5a6dfa0, 0xc007853eb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc006d60900, 0x5a4e420, 0xc011fc9380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc011fc9380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00f523a70, 0x5a4e420, 0xc011fc9380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc011fc9380, 0xc02a3fd1f0, 0xc00992a000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc00f523a70, 0x5a4e420, 0xc011fc9380, 0xc00992a000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc00f523a70, 0x5a4e420, 0xc011fc9380, 0xc00992a000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc00dc53828, 0x5a4e420, 0xc011fc9380, 0xc011deef00, 0xc00d21f7e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00dc53800, 0x5a4e420, 0xc011fc9380, 0xc011deef00, 0xc00d21f7e0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc0164014e0, 0x5a4e420, 0xc011fc9380, 0xc011deef00, 0xc00d21f7e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc0164014e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc02a3fd588, 0x4bfc8f0, 0x5, 0x5a50860, 0xc0164014e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc02a3fd588, 0x5a50860, 0xc0164014e0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc02a3fd588, 0x5a50860, 0xc0164014e0, 0xc, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc011fc9380, 0xc011deef00, 0xc00d21f7e0, 0x5a50860, 0xc0164014e0, 0x5a4e420, 0xc011fc9380)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc00fe4a000, 0x5a4e420, 0xc011fc9380)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc01237d600)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc02a3fd768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc013cdcc00, 0x5a4e420, 0xc011fc9380)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc013cdcc00, 0x5a4e420, 0xc011fc9380)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc01f4f8120, 0x5a4e420, 0xc011fc9380, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc012287500, 0xc00f523a70, 0xc0107a5700, 0xc00de60000, 0xc00d21f9f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc008f1d838, 0xc00d21f9f8, 0xc012287500, 0xc00f523a70, 0x5a50860, 0xc0164014e0, 0x0, 0xc00de60000, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00d21f300, 0x5a6dfa0, 0xc008f1d838, 0xc00d21f7e0, 0x2, 0x7f8b0dd7dcb0, 0xc010480f80, 0xc02a3fe000, 0xc015578478, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00d21f300, 0x5a6dfa0, 0xc008f1d838, 0xc00d21f7e0, 0x7f8b0dd7dcb0, 0xc010480f80, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00d21f300, 0x5a6dfa0, 0xc008f1d838, 0x5a6dda0, 0xc023796b00, 0xc0080c2bb6, 0x1e9, 0x11, 0x1, 0xc0095fda40, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00d21f300, 0x5a4e420, 0xc013ebbb40, 0x5a6dda0, 0xc023796b00, 0xc0080c2bb6, 0x1e9, 0x11, 0x1, 0xc0095fda40, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00d21f300, 0x5a4e420, 0xc013ebbb40, 0xc0095fda40, 0xc023789560, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc0080c2f2b, 0x0, 0x0, 0x1873785a, 0xed85aa647, 0x0, 0xc00d21f300, 0xc02a3ffa20, 0xc02a3ffa10, 0x37, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00d21f300, 0x5a4e420, 0xc013ebbb40, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00d21f300, 0x5a4e420, 0xc015206840, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc003dfc360, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc015206840, 0xc00d21f300, 0x5400, 0x15000, 0xc001805a40, 0xc003dfc360, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc0199f2ab1, 0xc00ee2c540, 0x5a4e420, 0xc015206840, 0xc003dfc360, 0xc001a38000, 0xc010480900, 0x5ab5540, 0xc019dcb880, 0xc0091b5ec0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 217970207 [semacquire, 13 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01fdbbb00, 0x1688c68bfdadc8ca, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01fdbbb00, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01fdbbad0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01fdbbad0, 0x4c245c5, 0x10, 0xc01268f6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01fdbbad0, 0xc0213f8280, 0xc01fdbbad0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01fdbba70, 0xc0213f8280, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01fdbba70, 0x4bc7440, 0xc0213f8280, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01fdbba70, 0x4bc7440, 0xc0213f8280, 0xc01fcad880, 0xc01fcad900, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01fdbba70, 0x4bc7440, 0xc0213f8280, 0xc01268f990, 0xca72c8, 0x48dd840, 0xc012875340)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01fdbba70, 0x4bc7440, 0xc0213f8280, 0xc01fcad880, 0xc012875340, 0xc012875340, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01fdbba70, 0x4bc7440, 0xc0213f8280, 0x0, 0xc00009c800, 0x0, 0xc027ecaa80)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01fdbba70, 0x4bc7440, 0xc0213f8280, 0xc01fcad880, 0xc012875300, 0xc01268fac8, 0xca72c8, 0x48dd840, 0xc012875300)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01fdbba70, 0x4bc7440, 0xc0213f8280, 0xc01fcad880, 0xc01fcad900, 0xc001878ba0, 0x661046, 0x491f620, 0xc01fdbba70)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01fdbba70, 0xc027ecaa20, 0xc001681820, 0x5a4e4e0, 0xc01fdbba70, 0xc0122a3800, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0122a0200, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0122a0200, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0122a0200)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217757945 [semacquire, 31 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc003149eb8, 0x1688c58a9370d04c, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc003149eb8, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0163d9110, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0163d9110, 0x4c245c5, 0x10, 0xc026e503f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0163d9110, 0xc00c8770e0, 0xc0163d9110, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0163d90e0, 0xc00c8770e0, 0xc003149838, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00e438c40, 0x5a4e4e0, 0xc0163d90e0, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00e438c40, 0x5a6dfa0, 0xc003149838, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc003149838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc003149838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc003149838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc003149838, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc003149698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0123011d0, 0x5a6dfa0, 0xc003149698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc012301198, 0x5a6dfa0, 0xc003149698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc012301168, 0x5a6dfa0, 0xc003149698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc0123010c8, 0x5a6dfa0, 0xc003149698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc0123010c8, 0x5a6dfa0, 0xc003149698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc012300ff8, 0x5a6dfa0, 0xc003149698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc012300fd8, 0x5a6dfa0, 0xc003149698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc012300f40, 0x5a6dfa0, 0xc003149698, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc012300d80, 0x5a4e420, 0xc0213b53c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc0213b53c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc011c15e60, 0x5a4e420, 0xc0213b53c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc0213b53c0, 0xc026e551f0, 0xc00f2f0a00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc011c15e60, 0x5a4e420, 0xc0213b53c0, 0xc00f2f0a00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc011c15e60, 0x5a4e420, 0xc0213b53c0, 0xc00f2f0a00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc022ba1828, 0x5a4e420, 0xc0213b53c0, 0xc0091bd500, 0xc016619de0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc022ba1800, 0x5a4e420, 0xc0213b53c0, 0xc0091bd500, 0xc016619de0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc003f728e0, 0x5a4e420, 0xc0213b53c0, 0xc0091bd500, 0xc016619de0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc003f728e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc026e55588, 0x4bfc8f0, 0x5, 0x5a50860, 0xc003f728e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc026e55588, 0x5a50860, 0xc003f728e0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc026e55588, 0x5a50860, 0xc003f728e0, 0xc, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc0213b53c0, 0xc0091bd500, 0xc016619de0, 0x5a50860, 0xc003f728e0, 0x5a4e420, 0xc0213b53c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc00a8be600, 0x5a4e420, 0xc0213b53c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc0129b4d00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc026e55768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc011c48000, 0x5a4e420, 0xc0213b53c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc011c48000, 0x5a4e420, 0xc0213b53c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc0072c0000, 0x5a4e420, 0xc0213b53c0, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc003205420, 0xc011c15e60, 0xc00cabdc00, 0xc0057ace00, 0xc016619ff8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc00212b9d8, 0xc016619ff8, 0xc003205420, 0xc011c15e60, 0x5a50860, 0xc003f728e0, 0x0, 0xc0057ace00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc016619900, 0x5a6dfa0, 0xc00212b9d8, 0xc016619de0, 0x2, 0x7f8b0dd7dcb0, 0xc012df6180, 0xc026e56000, 0xc01d727318, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc016619900, 0x5a6dfa0, 0xc00212b9d8, 0xc016619de0, 0x7f8b0dd7dcb0, 0xc012df6180, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc016619900, 0x5a6dfa0, 0xc00212b9d8, 0x5a6dda0, 0xc00cabc780, 0xc00f84a96c, 0x1e9, 0x11, 0x1, 0xc0303742d0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc016619900, 0x5a4e420, 0xc019587300, 0x5a6dda0, 0xc00cabc780, 0xc00f84a96c, 0x1e9, 0x11, 0x1, 0xc0303742d0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc016619900, 0x5a4e420, 0xc019587300, 0xc0303742d0, 0xc00505ed80, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc00f84acd3, 0x0, 0x0, 0x25f51fb2, 0xed85aa680, 0x0, 0xc016619900, 0xc026e57a20, 0xc026e57a10, 0x25, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc016619900, 0x5a4e420, 0xc019587300, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc016619900, 0x5a4e420, 0xc00d319280, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc01dc0b950, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc00d319280, 0xc016619900, 0x5400, 0x15000, 0xc001805a40, 0xc01dc0b950, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc01181f896, 0xc008c77380, 0x5a4e420, 0xc00d319280, 0xc01dc0b950, 0xc001a38000, 0xc012df5b00, 0x5ab5540, 0xc00f20b480, 0xc00a88d7a0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216776934 [semacquire, 134 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc019f91fb0, 0x1688bfecf207ac2a, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc019f91fb0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc019f91ec0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc019f91ec0, 0x4c245c5, 0x10, 0xc0059d16c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc019f91ec0, 0xc0024f7400, 0xc019f91ec0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc019f91e30, 0xc0024f7400, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc019f91e30, 0x4bc7440, 0xc0024f7400, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc019f91e30, 0x4bc7440, 0xc0024f7400, 0xc01ed26280, 0xc01ed262a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc019f91e30, 0x4bc7440, 0xc0024f7400, 0xc0059d1990, 0xca72c8, 0x48dd840, 0xc004f8fc00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc019f91e30, 0x4bc7440, 0xc0024f7400, 0xc01ed26280, 0xc004f8fc00, 0xc004f8fc00, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc019f91e30, 0x4bc7440, 0xc0024f7400, 0x0, 0xc00e8e4800, 0x0, 0xc00a9ff260)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc019f91e30, 0x4bc7440, 0xc0024f7400, 0xc01ed26280, 0xc004f8fb80, 0xc0059d1ac8, 0xca72c8, 0x48dd840, 0xc004f8fb80)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc019f91e30, 0x4bc7440, 0xc0024f7400, 0xc01ed26280, 0xc01ed262a0, 0xc01e93eba0, 0x661046, 0x491f620, 0xc019f91e30)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc019f91e30, 0xc00a9ff1a0, 0xc001681820, 0x5a4e4e0, 0xc019f91e30, 0xc00ec7b200, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc014005500, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc014005500, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc014005500)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216790157 [semacquire, 133 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01e951080, 0x1688bffc146431ff, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01e951080, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01e951020, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01e951020, 0x4c245c5, 0x10, 0xc005ced6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01e951020, 0xc010835720, 0xc01e951020, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01e950f60, 0xc010835720, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01e950f60, 0x4bc7440, 0xc010835720, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01e950f60, 0x4bc7440, 0xc010835720, 0xc01fff5020, 0xc01fff5040, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01e950f60, 0x4bc7440, 0xc010835720, 0xc005ced990, 0xca72c8, 0x48dd840, 0xc004fed4c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01e950f60, 0x4bc7440, 0xc010835720, 0xc01fff5020, 0xc004fed4c0, 0xc004fed4c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01e950f60, 0x4bc7440, 0xc010835720, 0x0, 0xc000100400, 0x0, 0xc01f1762a0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01e950f60, 0x4bc7440, 0xc010835720, 0xc01fff5020, 0xc004fed480, 0xc005cedac8, 0xca72c8, 0x48dd840, 0xc004fed480)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01e950f60, 0x4bc7440, 0xc010835720, 0xc01fff5020, 0xc01fff5040, 0xc0072a2ba0, 0x661046, 0x491f620, 0xc01e950f60)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01e950f60, 0xc01f176240, 0xc001681820, 0x5a4e4e0, 0xc01e950f60, 0xc00a33c000, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0194b3d00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0194b3d00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc0194b3d00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216085772 [chan receive, 105 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc002334400, 0x5a4e4e0, 0xc0190d2a50, 0xc01eec5ac0, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc0190d2a50, 0x5ab2120, 0xc01f501c00, 0xc0065dc028, 0xd, 0xc0190d2a20, 0x59fb820, 0xc015cbc5d0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc0190d2a50, 0x5ab21e0, 0xc00da85e00, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc00da85e00)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc00da85e00, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 217672019 [semacquire, 43 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc0214cbec0, 0x1688c4ed8b71e49b, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc0214cbec0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0214cbe90, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0214cbe90, 0x4c245c5, 0x10, 0xc0040a36c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0214cbe90, 0xc016365540, 0xc0214cbe90, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0214cbda0, 0xc016365540, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0214cbda0, 0x4bc7440, 0xc016365540, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0214cbda0, 0x4bc7440, 0xc016365540, 0xc01677f2c0, 0xc01677f2e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0214cbda0, 0x4bc7440, 0xc016365540, 0xc0040a3990, 0xca72c8, 0x48dd840, 0xc0191290c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0214cbda0, 0x4bc7440, 0xc016365540, 0xc01677f2c0, 0xc0191290c0, 0xc0191290c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0214cbda0, 0x4bc7440, 0xc016365540, 0x0, 0x87067e0, 0x0, 0xc01f14bc20)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0214cbda0, 0x4bc7440, 0xc016365540, 0xc01677f2c0, 0xc019129080, 0xc0040a3ac8, 0xca72c8, 0x48dd840, 0xc019129080)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0214cbda0, 0x4bc7440, 0xc016365540, 0xc01677f2c0, 0xc01677f2e0, 0xc0020f6ba0, 0x661046, 0x491f620, 0xc0214cbda0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0214cbda0, 0xc01f14bbc0, 0xc001681820, 0x5a4e4e0, 0xc0214cbda0, 0xc00bcbec00, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc003e14600, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc003e14600, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc003e14600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217749609 [semacquire, 32 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc0073ca178, 0x1688c57e7fbe8ec6, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc0073ca178, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01516f800, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01516f800, 0x4c245c5, 0x10, 0xc02a2a83f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01516f800, 0xc013ecabe0, 0xc01516f800, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01516f7d0, 0xc013ecabe0, 0xc0065afd18, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc012950200, 0x5a4e4e0, 0xc01516f7d0, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc012950200, 0x5a6dfa0, 0xc0065afd18, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc0065afd18, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc0065afd18, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc0065afd18, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc0065afd18, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc0065afb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc019d53f50, 0x5a6dfa0, 0xc0065afb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc019d53f18, 0x5a6dfa0, 0xc0065afb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc019d53ee8, 0x5a6dfa0, 0xc0065afb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc019d53e48, 0x5a6dfa0, 0xc0065afb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc019d53e48, 0x5a6dfa0, 0xc0065afb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc019d53d78, 0x5a6dfa0, 0xc0065afb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc019d53d58, 0x5a6dfa0, 0xc0065afb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc019d53cc0, 0x5a6dfa0, 0xc0065afb78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc019d53b00, 0x5a4e420, 0xc01b25a540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc01b25a540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc010597320, 0x5a4e420, 0xc01b25a540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc01b25a540, 0xc02a2ad1f0, 0xc00d190000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc010597320, 0x5a4e420, 0xc01b25a540, 0xc00d190000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc010597320, 0x5a4e420, 0xc01b25a540, 0xc00d190000, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc00f8b5828, 0x5a4e420, 0xc01b25a540, 0xc015198600, 0xc005f27de0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00f8b5800, 0x5a4e420, 0xc01b25a540, 0xc015198600, 0xc005f27de0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc0116e64c0, 0x5a4e420, 0xc01b25a540, 0xc015198600, 0xc005f27de0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc0116e64c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc02a2ad588, 0x4bfc8f0, 0x5, 0x5a50860, 0xc0116e64c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc01a7dd588, 0x5a50860, 0xc0116e64c0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc01a7dd588, 0x5a50860, 0xc0116e64c0, 0xc, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc01b25a540, 0xc015198600, 0xc005f27de0, 0x5a50860, 0xc0116e64c0, 0x5a4e420, 0xc01b25a540)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc00cd16c00, 0x5a4e420, 0xc01b25a540)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc020e72400)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc01a7dd768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc02480a000, 0x5a4e420, 0xc01b25a540)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc02480a000, 0x5a4e420, 0xc01b25a540)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc01653bc20, 0x5a4e420, 0xc01b25a540, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc012287180, 0xc010597320, 0xc005b7b800, 0xc0223dca80, 0xc005f27ff8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc006606e78, 0xc005f27ff8, 0xc012287180, 0xc010597320, 0x5a50860, 0xc0116e64c0, 0x0, 0xc0223dca80, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc005f27900, 0x5a6dfa0, 0xc006606e78, 0xc005f27de0, 0x2, 0x7f8b0dd7dcb0, 0xc00ddd2180, 0xc01a7de000, 0xc01ee10dd8, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc005f27900, 0x5a6dfa0, 0xc006606e78, 0xc005f27de0, 0x7f8b0dd7dcb0, 0xc00ddd2180, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc005f27900, 0x5a6dfa0, 0xc006606e78, 0x5a6dda0, 0xc01fcc0580, 0xc007151030, 0x1e9, 0x11, 0x1, 0xc00f7c53b0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc005f27900, 0x5a4e420, 0xc0161da7c0, 0x5a6dda0, 0xc01fcc0580, 0xc007151030, 0x1e9, 0x11, 0x1, 0xc00f7c53b0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc005f27900, 0x5a4e420, 0xc0161da7c0, 0xc00f7c53b0, 0xc01653afc0, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc0071513a7, 0x0, 0x0, 0x2db2b56d, 0xed85aa64c, 0x0, 0xc005f27900, 0xc02a2afa20, 0xc02a2afa10, 0x5, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc005f27900, 0x5a4e420, 0xc0161da7c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc005f27900, 0x5a4e420, 0xc0128eb340, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc00c993870, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc0128eb340, 0xc005f27900, 0x5400, 0x15000, 0xc001805a40, 0xc00c993870, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc01adff4a6, 0xc01ee3fac0, 0x5a4e420, 0xc0128eb340, 0xc00c993870, 0xc001a38000, 0xc00ddd1b00, 0x5ab5540, 0xc003932f80, 0xc00f5e0840, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 216090887 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc021113a10, 0x1688bcc27feaecf9, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc021113a10, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0211139e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0211139e0, 0x4c245c5, 0x10, 0xc02148f6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0211139e0, 0xc02114c8c0, 0xc0211139e0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc021113980, 0xc02114c8c0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc021113980, 0x4bc7440, 0xc02114c8c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc021113980, 0x4bc7440, 0xc02114c8c0, 0xc014940e00, 0xc014940e20, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc021113980, 0x4bc7440, 0xc02114c8c0, 0xc02148f990, 0xca72c8, 0x48dd840, 0xc00ac1b380)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc021113980, 0x4bc7440, 0xc02114c8c0, 0xc014940e00, 0xc00ac1b380, 0xc00ac1b380, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc021113980, 0x4bc7440, 0xc02114c8c0, 0x0, 0xc000600000, 0x0, 0xc01acd9200)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc021113980, 0x4bc7440, 0xc02114c8c0, 0xc014940e00, 0xc00ac1b340, 0xc02148fac8, 0xca72c8, 0x48dd840, 0xc00ac1b340)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc021113980, 0x4bc7440, 0xc02114c8c0, 0xc014940e00, 0xc014940e20, 0xc00252eba0, 0x661046, 0x491f620, 0xc021113980)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc021113980, 0xc01acd91a0, 0xc001681820, 0x5a4e4e0, 0xc021113980, 0xc021151800, 0x334)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc020ddc600, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc020ddc600, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc020ddc600)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217474963 [semacquire, 61 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc00d1bd9e0, 0x1688c3e8c4f00517, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc00d1bd9e0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc00d1bd950, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc00d1bd950, 0x4c245c5, 0x10, 0xc0087876c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc00d1bd950, 0xc01e8461e0, 0xc00d1bd950, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc00d1bd8f0, 0xc01e8461e0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc00d1bd8f0, 0x4bc7440, 0xc01e8461e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc00d1bd8f0, 0x4bc7440, 0xc01e8461e0, 0xc01aa9a120, 0xc01aa9a140, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00d1bd8f0, 0x4bc7440, 0xc01e8461e0, 0xc008787990, 0xca72c8, 0x48dd840, 0xc0169cb680)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc00d1bd8f0, 0x4bc7440, 0xc01e8461e0, 0xc01aa9a120, 0xc0169cb680, 0xc0169cb680, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc00d1bd8f0, 0x4bc7440, 0xc01e8461e0, 0x0, 0xc003d76000, 0x0, 0xc010ef0180)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc00d1bd8f0, 0x4bc7440, 0xc01e8461e0, 0xc01aa9a120, 0xc0169cb640, 0xc008787ac8, 0xca72c8, 0x48dd840, 0xc0169cb640)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc00d1bd8f0, 0x4bc7440, 0xc01e8461e0, 0xc01aa9a120, 0xc01aa9a140, 0xc0013eeba0, 0x661046, 0x491f620, 0xc00d1bd8f0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc00d1bd8f0, 0xc010ef0120, 0xc001681820, 0x5a4e4e0, 0xc00d1bd8f0, 0xc00db4cc00, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc009c06800, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc009c06800, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc009c06800)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216997763 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01abb85d0, 0x1688c18c702a18d7, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01abb85d0, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01abb85a0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01abb85a0, 0x4c245c5, 0x10, 0xc005a156c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01abb85a0, 0xc01fc663c0, 0xc01abb85a0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01abb8540, 0xc01fc663c0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01abb8540, 0x4bc7440, 0xc01fc663c0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01abb8540, 0x4bc7440, 0xc01fc663c0, 0xc01fb8b9e0, 0xc01fb8ba00, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01abb8540, 0x4bc7440, 0xc01fc663c0, 0xc005a15990, 0xca72c8, 0x48dd840, 0xc00dd25040)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01abb8540, 0x4bc7440, 0xc01fc663c0, 0xc01fb8b9e0, 0xc00dd25040, 0xc00dd25040, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01abb8540, 0x4bc7440, 0xc01fc663c0, 0x0, 0xc0002a9800, 0x0, 0xc00ff1ff20)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01abb8540, 0x4bc7440, 0xc01fc663c0, 0xc01fb8b9e0, 0xc00dd25000, 0xc005a15ac8, 0xca72c8, 0x48dd840, 0xc00dd25000)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01abb8540, 0x4bc7440, 0xc01fc663c0, 0xc01fb8b9e0, 0xc01fb8ba00, 0xc02eb7bba0, 0x661046, 0x491f620, 0xc01abb8540)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01abb8540, 0xc00ff1fec0, 0xc001681820, 0x5a4e4e0, 0xc01abb8540, 0xc00eecb800, 0x350)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00bfc5000, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00bfc5000, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc00bfc5000)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 217756768 [semacquire, 31 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc00212a178, 0x1688c5896240e0e6, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc00212a178, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc0015d16b0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc0015d16b0, 0x4c245c5, 0x10, 0xc021f263f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc0015d16b0, 0xc01ab94320, 0xc0015d16b0, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0015d1680, 0xc01ab94320, 0xc00511db78, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00f5e4280, 0x5a4e4e0, 0xc0015d1680, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00f5e4280, 0x5a6dfa0, 0xc00511db78, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc00511db78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc00511db78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc00511db78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc00511db78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc00511d9d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00d439650, 0x5a6dfa0, 0xc00511d9d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00d439618, 0x5a6dfa0, 0xc00511d9d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00d4395e8, 0x5a6dfa0, 0xc00511d9d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00d439548, 0x5a6dfa0, 0xc00511d9d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00d439548, 0x5a6dfa0, 0xc00511d9d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00d439478, 0x5a6dfa0, 0xc00511d9d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00d439458, 0x5a6dfa0, 0xc00511d9d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00d4393c0, 0x5a6dfa0, 0xc00511d9d8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00d439200, 0x5a4e420, 0xc015eb5980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc015eb5980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc00682d7a0, 0x5a4e420, 0xc015eb5980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc015eb5980, 0xc021f2b1f0, 0xc00ac27400, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc00682d7a0, 0x5a4e420, 0xc015eb5980, 0xc00ac27400, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc00682d7a0, 0x5a4e420, 0xc015eb5980, 0xc00ac27400, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc00e74b228, 0x5a4e420, 0xc015eb5980, 0xc011116000, 0xc01179bde0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00e74b200, 0x5a4e420, 0xc015eb5980, 0xc011116000, 0xc01179bde0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc01e2131e0, 0x5a4e420, 0xc015eb5980, 0xc011116000, 0xc01179bde0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc01e2131e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc021f2b588, 0x4bfc8f0, 0x5, 0x5a50860, 0xc01e2131e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc021f2b588, 0x5a50860, 0xc01e2131e0, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc021f2b588, 0x5a50860, 0xc01e2131e0, 0xc, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc015eb5980, 0xc011116000, 0xc01179bde0, 0x5a50860, 0xc01e2131e0, 0x5a4e420, 0xc015eb5980)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc00e74b800, 0x5a4e420, 0xc015eb5980)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc00b9c7b00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc021f2b768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc01b1e8600, 0x5a4e420, 0xc015eb5980)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc01b1e8600, 0x5a4e420, 0xc015eb5980)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc011f76a20, 0x5a4e420, 0xc015eb5980, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc01a70e2a0, 0xc00682d7a0, 0xc0136d2a00, 0xc00a715500, 0xc01179bff8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc005e3d698, 0xc01179bff8, 0xc01a70e2a0, 0xc00682d7a0, 0x5a50860, 0xc01e2131e0, 0x0, 0xc00a715500, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc01179b900, 0x5a6dfa0, 0xc005e3d698, 0xc01179bde0, 0x2, 0x7f8b0dd7dcb0, 0xc0028ed880, 0xc021f2c000, 0xc008d410d8, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc01179b900, 0x5a6dfa0, 0xc005e3d698, 0xc01179bde0, 0x7f8b0dd7dcb0, 0xc0028ed880, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc01179b900, 0x5a6dfa0, 0xc005e3d698, 0x5a6dda0, 0xc0136d2100, 0xc01ee553ce, 0x1e9, 0x11, 0x1, 0xc00826a4b0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc01179b900, 0x5a4e420, 0xc01aacd880, 0x5a6dda0, 0xc0136d2100, 0xc01ee553ce, 0x1e9, 0x11, 0x1, 0xc00826a4b0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc01179b900, 0x5a4e420, 0xc01aacd880, 0xc00826a4b0, 0xc006703e60, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc01ee55734, 0x0, 0x0, 0x1ecae6ef, 0xed85aa67b, 0x0, 0xc01179b900, 0xc021f2da20, 0xc021f2da10, 0xbf, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc01179b900, 0x5a4e420, 0xc01aacd880, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc01179b900, 0x5a4e420, 0xc00cd52940, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc00c9164f0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc00cd52940, 0xc01179b900, 0x5400, 0x15000, 0xc001805a40, 0xc00c9164f0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc021040236, 0xc019afca20, 0x5a4e420, 0xc00cd52940, 0xc00c9164f0, 0xc001a38000, 0xc0028ed200, 0x5ab5540, 0xc00da98580, 0xc0132adda0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 218080013 [chan receive]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc009e55200, 0x5a4e4e0, 0xc02a8bdec0, 0xc0034c8c30, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc02a8bdec0, 0x5ab2120, 0xc013975180, 0xc01ec66028, 0xd, 0xc02a8bde90, 0x59fb820, 0xc029f3db90, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc02a8bdec0, 0x5ab21e0, 0xc011d3c420, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc011d3c420)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc011d3c420, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 217396772 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc006d47260, 0x1688c384a676c37e, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc006d47260, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc006d47230, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc006d47230, 0x4c245c5, 0x10, 0xc0086b16c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc006d47230, 0xc00e475f40, 0xc006d47230, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc006d471d0, 0xc00e475f40, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc006d471d0, 0x4bc7440, 0xc00e475f40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc006d471d0, 0x4bc7440, 0xc00e475f40, 0xc0161dfb20, 0xc0161dfb40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc006d471d0, 0x4bc7440, 0xc00e475f40, 0xc0086b1990, 0xca72c8, 0x48dd840, 0xc00ec03140)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc006d471d0, 0x4bc7440, 0xc00e475f40, 0xc0161dfb20, 0xc00ec03140, 0xc00ec03140, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc006d471d0, 0x4bc7440, 0xc00e475f40, 0x0, 0xc00fba5800, 0x0, 0xc007501740)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc006d471d0, 0x4bc7440, 0xc00e475f40, 0xc0161dfb20, 0xc00ec03100, 0xc0086b1ac8, 0xca72c8, 0x48dd840, 0xc00ec03100)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc006d471d0, 0x4bc7440, 0xc00e475f40, 0xc0161dfb20, 0xc0161dfb40, 0xc00190aba0, 0x661046, 0x491f620, 0xc006d471d0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc006d471d0, 0xc0075016e0, 0xc001681820, 0x5a4e4e0, 0xc006d471d0, 0xc00785a600, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007764300, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007764300, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc007764300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218080677 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc023a2f698, 0x1688c7284e65a88b, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc023a2f698, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01d719830, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01d719830, 0x4c245c5, 0x10, 0xc0285163f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01d719830, 0xc024080960, 0xc01d719830, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01d719800, 0xc024080960, 0xc023a2f358, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc01af77440, 0x5a4e4e0, 0xc01d719800, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc01af77440, 0x5a6dfa0, 0xc023a2f358, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc023a2f358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc023a2f358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc023a2f358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc023a2f358, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc023a2f1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc02aa17650, 0x5a6dfa0, 0xc023a2f1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc02aa17618, 0x5a6dfa0, 0xc023a2f1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc02aa175e8, 0x5a6dfa0, 0xc023a2f1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc02aa17548, 0x5a6dfa0, 0xc023a2f1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc02aa17548, 0x5a6dfa0, 0xc023a2f1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc02aa17478, 0x5a6dfa0, 0xc023a2f1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc02aa17458, 0x5a6dfa0, 0xc023a2f1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc02aa173c0, 0x5a6dfa0, 0xc023a2f1b8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc02aa17200, 0x5a4e420, 0xc01d731c40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc01d731c40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc0230d3ef0, 0x5a4e420, 0xc01d731c40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc01d731c40, 0xc02851b1f0, 0xc00cf20a00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc0230d3ef0, 0x5a4e420, 0xc01d731c40, 0xc00cf20a00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc0230d3ef0, 0x5a4e420, 0xc01d731c40, 0xc00cf20a00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc023473228, 0x5a4e420, 0xc01d731c40, 0xc023382000, 0xc022255de0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc023473200, 0x5a4e420, 0xc01d731c40, 0xc023382000, 0xc022255de0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc00f774320, 0x5a4e420, 0xc01d731c40, 0xc023382000, 0xc022255de0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x4bfc8f0, 0x5, 0x5a50860, 0xc00f774320, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:498 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1(0xc02851b588, 0x4bfc8f0, 0x5, 0x5a50860, 0xc00f774320)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:119 +0x5d
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc02851b588, 0x5a50860, 0xc00f774320, 0x4bfc8f0, 0x5)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:296 +0x188
github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc02851b588, 0x5a50860, 0xc00f774320, 0xc, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:86 +0x8f
github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:50
github.com/cockroachdb/cockroach/pkg/sql.startExec(0x5a4e420, 0xc01d731c40, 0xc023382000, 0xc022255de0, 0x5a50860, 0xc00f774320, 0x5a4e420, 0xc01d731c40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:501 +0x11f
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc023473800, 0x5a4e420, 0xc01d731c40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:121 +0xb6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc023990900)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:156 +0x113
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc02851b768, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91 +0x57
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Start(0xc022104600, 0x5a4e420, 0xc01d731c40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:273 +0xc5
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).Run(0xc022104600, 0x5a4e420, 0xc01d731c40)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:780 +0x52
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc00aabad80, 0x5a4e420, 0xc01d731c40, 0x52f7fd8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:384 +0x2e2
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc0017bd3f0, 0xc002606ee0, 0xc0230d3ef0, 0xc0237ec900, 0xc0229cca80, 0xc022255ff8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:367 +0x471
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc0017bd3f0, 0x5a6dfa0, 0xc023449b78, 0xc022255ff8, 0xc002606ee0, 0xc0230d3ef0, 0x5a50860, 0xc00f774320, 0x0, 0xc0229cca80, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1011 +0x1c8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc022255900, 0x5a6dfa0, 0xc023449b78, 0xc022255de0, 0x2, 0x7f8b0dd7dcb0, 0xc0232fc680, 0xc02851c000, 0xc023225c18, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1055 +0x468
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc022255900, 0x5a6dfa0, 0xc023449b78, 0xc022255de0, 0x7f8b0dd7dcb0, 0xc0232fc680, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:917 +0x75b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc022255900, 0x5a6dfa0, 0xc023449b78, 0x5a6dda0, 0xc0237ec200, 0xc023456bf1, 0x1e9, 0x11, 0x1, 0xc022abbc20, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:661 +0xf86
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc022255900, 0x5a4e420, 0xc022717040, 0x5a6dda0, 0xc0237ec200, 0xc023456bf1, 0x1e9, 0x11, 0x1, 0xc022abbc20, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:123 +0xb14
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc022255900, 0x5a4e420, 0xc022717040, 0xc022abbc20, 0xc01e76bd40, 0x11, 0x11, 0x875b27c, 0x1, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:210 +0x14e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc023456f5b, 0x0, 0x0, 0x2375cb9d, 0xed85aad71, 0x0, 0xc022255900, 0xc02851da20, 0xc02851da10, 0xa2, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1596 +0x47b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc022255900, 0x5a4e420, 0xc022717040, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1598 +0x45c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc022255900, 0x5a4e420, 0xc020fc52c0, 0xc0018059a0, 0x5400, 0x15000, 0xc001805a40, 0xc00bb855c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1450 +0x228
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc001a38000, 0x5a4e420, 0xc020fc52c0, 0xc022255900, 0x5400, 0x15000, 0xc001805a40, 0xc00bb855c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:484 +0xce
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc00c9bd071, 0xc01a0b1a20, 0x5a4e420, 0xc020fc52c0, 0xc00bb855c0, 0xc001a38000, 0xc0232fc000, 0x5ab5540, 0xc0232a8400, 0xc00d1faba0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:626 +0x3ea
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:554 +0x185
goroutine 218060466 [semacquire, 4 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc023375950, 0x1688c70d36e24a17, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc023375950, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc023375920, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc023375920, 0x4c245c5, 0x10, 0xc01218b6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc023375920, 0xc02974e0a0, 0xc023375920, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc0233758c0, 0xc02974e0a0, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc0233758c0, 0x4bc7440, 0xc02974e0a0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc0233758c0, 0x4bc7440, 0xc02974e0a0, 0xc0162998e0, 0xc016299900, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0233758c0, 0x4bc7440, 0xc02974e0a0, 0xc01218b990, 0xca72c8, 0x48dd840, 0xc0111f7b00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc0233758c0, 0x4bc7440, 0xc02974e0a0, 0xc0162998e0, 0xc0111f7b00, 0xc0111f7b00, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc0233758c0, 0x4bc7440, 0xc02974e0a0, 0x0, 0xc003d76000, 0x0, 0xc01012d8c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc0233758c0, 0x4bc7440, 0xc02974e0a0, 0xc0162998e0, 0xc0111f7ac0, 0xc01218bac8, 0xca72c8, 0x48dd840, 0xc0111f7ac0)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc0233758c0, 0x4bc7440, 0xc02974e0a0, 0xc0162998e0, 0xc016299900, 0xc02f07cba0, 0x661046, 0x491f620, 0xc0233758c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc0233758c0, 0xc01012d860, 0xc001681820, 0x5a4e4e0, 0xc0233758c0, 0xc0112df200, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00fb7fd00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00fb7fd00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00fb7fd00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 218082038 [semacquire, 2 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc02a02fc20, 0x1688c72a28af0ad3, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc02a02fc20, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc02a02fbf0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc02a02fbf0, 0x4c245c5, 0x10, 0xc02a5576c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc02a02fbf0, 0xc029841f40, 0xc02a02fbf0, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc02a02fb60, 0xc029841f40, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc02a02fb60, 0x4bc7440, 0xc029841f40, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc02a02fb60, 0x4bc7440, 0xc029841f40, 0xc0214159c0, 0xc0214159e0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02a02fb60, 0x4bc7440, 0xc029841f40, 0xc02a557990, 0xca72c8, 0x48dd840, 0xc0119ea9c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc02a02fb60, 0x4bc7440, 0xc029841f40, 0xc0214159c0, 0xc0119ea9c0, 0xc0119ea9c0, 0x4, 0xc001abb890, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc02a02fb60, 0x4bc7440, 0xc029841f40, 0x0, 0xc0002a9800, 0x0, 0xc02a00b9e0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc02a02fb60, 0x4bc7440, 0xc029841f40, 0xc0214159c0, 0xc0119ea980, 0xc02a557ac8, 0xca72c8, 0x48dd840, 0xc0119ea980)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc02a02fb60, 0x4bc7440, 0xc029841f40, 0xc0214159c0, 0xc0214159e0, 0xc0072a0ba0, 0x661046, 0x491f620, 0xc02a02fb60)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc02a02fb60, 0xc02a00b980, 0xc001681820, 0x5a4e4e0, 0xc02a02fb60, 0xc02a054000, 0x328)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc02985db00, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc02985db00, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002783830, 0xc0005601a0, 0x5aba220, 0xc0022b7c80, 0xc02985db00)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216343708 [semacquire, 171 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a4e4e0, 0xc01f991260, 0x1688bde48ef6e3a9, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a4e4e0, 0xc01f991260, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc01f991230, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc01f991230, 0x4c245c5, 0x10, 0xc002f9d6c0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc01f991230, 0xc00fece280, 0xc01f991230, 0x46cfe00, 0x4b7ca60)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc01f9911d0, 0xc00fece280, 0xc0058cac00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x5a4e4e0, 0xc01f9911d0, 0x4bc7440, 0xc00fece280, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9132 +0x89
github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1(0x5a4e4e0, 0xc01f9911d0, 0x4bc7440, 0xc00fece280, 0xc019cb8ac0, 0xc019cb8ae0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:126 +0x4a9
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01f9911d0, 0x4bc7440, 0xc00fece280, 0xc002f9d990, 0xca72c8, 0x48dd840, 0xc00fcdc6c0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1(0x5a4e4e0, 0xc01f9911d0, 0x4bc7440, 0xc00fece280, 0xc019cb8ac0, 0xc00fcdc6c0, 0xc00fcdc6c0, 0x4, 0xc00280c990, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:174 +0xa8
google.golang.org/grpc.getChainUnaryHandler.func1(0x5a4e4e0, 0xc01f9911d0, 0x4bc7440, 0xc00fece280, 0x0, 0x87067e0, 0x0, 0xc01a40a060)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0xe7
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor(0x5a4e4e0, 0xc01f9911d0, 0x4bc7440, 0xc00fece280, 0xc019cb8ac0, 0xc00fcdc680, 0xc002f9dac8, 0xca72c8, 0x48dd840, 0xc00fcdc680)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/auth.go:60 +0x8a
google.golang.org/grpc.chainUnaryServerInterceptors.func1(0x5a4e4e0, 0xc01f9911d0, 0x4bc7440, 0xc00fece280, 0xc019cb8ac0, 0xc019cb8ae0, 0xc0013f4ba0, 0x661046, 0x491f620, 0xc01f9911d0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:907 +0xd0
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x4b7ca60, 0xc0058cac00, 0x5a4e4e0, 0xc01f9911d0, 0xc01a40a000, 0xc001681820, 0x5a4e4e0, 0xc01f9911d0, 0xc00fb24c00, 0x6d)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9134 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00aec6300, 0xc000f57590, 0x7cb3780, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1082 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00aec6300, 0x0)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1405 +0xcc5
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc001d69550, 0xc0005601a0, 0x5aba220, 0xc002230780, 0xc00aec6300)
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:746 +0xa5
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:744 +0xa5
goroutine 216091378 [chan receive, 105 minutes]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc022148000, 0x5a4e4e0, 0xc0153e6fc0, 0xc00363b690, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc0153e6fc0, 0x5ab2120, 0xc016894380, 0xc0198c5009, 0xd, 0xc0153e6f90, 0x59fb820, 0xc00da22210, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc0153e6fc0, 0x5ab21e0, 0xc01594f9e0, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc01594f9e0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc01594f9e0, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 216262863 [chan receive]:
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc002b17b00, 0x5a4e4e0, 0xc01a628ae0, 0xc01ee76aa0, 0xc001a38000, 0x5400, 0x15000, 0xc001805a40, 0x0, 0x4, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:487 +0x5e6
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc001861900, 0x5a4e4e0, 0xc01a628ae0, 0x5ab2120, 0xc011aa2e00, 0xc009144009, 0x6, 0xc01a628ab0, 0x59fb820, 0xc019df6f30, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:153 +0x1b8
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc001861900, 0x5a4e4e0, 0xc01a628ae0, 0x5ab21e0, 0xc01a504ae0, 0x5a4e401, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:658 +0xc18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1(0x5ab21e0, 0xc01a504ae0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:2096 +0xf7
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc000c04b80, 0x5a4e4e0, 0xc000c14e70, 0xc006832040, 0x5ab21e0, 0xc01a504ae0, 0xc00d7e8780)
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:156 +0xaf
created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith
/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:153 +0x24a
goroutine 217748235 [semacquire, 32 minutes]:
sync.runtime_SemacquireMutex(0xc019f595fc, 0xc0018ad500, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).IsInitialized(0xc019f59200, 0x5c00)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_init.go:231 +0xb4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc0018ad000, 0x5a6dfa0, 0xc01156eb38, 0x1688c57cdc786a34, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:170 +0x3a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0002f7c00, 0x5a6dfa0, 0xc01156eb38, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:191 +0x118
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x5a4e4e0, 0xc014b66f90, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:875 +0x225
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000c04b80, 0x5a4e4e0, 0xc014b66f90, 0x4c245c5, 0x10, 0xc02105a3f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:330 +0xb2
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc0058cac00, 0x5a4e4e0, 0xc014b66f90, 0xc001edf180, 0xc014b66f90, 0x98, 0x497c4a0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:863 +0x165
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc0058cac00, 0x5a4e4e0, 0xc014b66f30, 0xc001edf180, 0xc01156e7f8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:901 +0xa5
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...)
/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:463
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc00cd539c0, 0x5a4e4e0, 0xc014b66f30, 0x1, 0x5a9b500, 0xc000cc0480, 0x0, 0x0, 0x100000001, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:193 +0x176
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc00cd539c0, 0x5a6dfa0, 0xc01156e7f8, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:175 +0x198
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00003a800, 0x5a6dfa0, 0xc01156e7f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c79, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1889 +0xbb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00003a800, 0x5a6dfa0, 0xc01156e7f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1507 +0x3a7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00003a800, 0x5a6dfa0, 0xc01156e7f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1291 +0xc85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc00003a800, 0x5a6dfa0, 0xc01156e7f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:886 +0xe85
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00003a800, 0x5a6dfa0, 0xc01156e658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:782 +0x994
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc0016ce450, 0x5a6dfa0, 0xc01156e658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc0016ce418, 0x5a6dfa0, 0xc01156e658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x9e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc0016ce3e8, 0x5a6dfa0, 0xc01156e658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x122
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc0016ce348, 0x5a6dfa0, 0xc01156e658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:267 +0xb5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc0016ce348, 0x5a6dfa0, 0xc01156e658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:202 +0x2e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc0016ce278, 0x5a6dfa0, 0xc01156e658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:255 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc0016ce258, 0x5a6dfa0, 0xc01156e658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x215
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0016ce1c0, 0x5a6dfa0, 0xc01156e658, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:171 +0x23f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc0016ce000, 0x5a4e420, 0xc00d6ef3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:513 +0x302
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc0002f7650, 0x5a4e420, 0xc00d6ef3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:809 +0x135
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc011c143f0, 0x5a4e420, 0xc00d6ef3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:951 +0x12e
github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x5a4e420, 0xc00d6ef3c0, 0xc02105f1f0, 0xc00f930f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:720 +0x127
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc011c143f0, 0x5a4e420, 0xc00d6ef3c0, 0xc00f930f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:620 +0x9c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).CommitInBatch(0xc011c143f0, 0x5a4e420, 0xc00d6ef3c0, 0xc00f930f00, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:682 +0x15e
github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc000aca028, 0x5a4e420, 0xc00d6ef3c0, 0xc00dff8000, 0xc02201d7e0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:155 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc000aca000, 0x5a4e420, 0xc00d6ef3c0, 0xc00dff8000, 0xc02201d7e0, 0x471c000, 0x1, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/update.go:193 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc01e81d3a0, 0x5a4e420, 0xc00d6ef3c0, 0xc00dff8000, 0xc02201d7e0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:159 +0xc3
github.com/cockroachdb/cockroach/pkg/sql.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment