Skip to content

Instantly share code, notes, and snippets.

@jsturtevant

jsturtevant/containerd

Last active August 3, 2021 23:19
Show Gist options
  • Save jsturtevant/a6ab989e03615940a349adbdab714e07 to your computer and use it in GitHub Desktop.
Save jsturtevant/a6ab989e03615940a349adbdab714e07 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
containerd
goroutine 3886 [select]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000440580, 0x22bf240, 0xc0008909c0, 0xc0005dc360, 0xc0000dd000, 0xc0009e9120, 0x5)
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:155 +0x18c
github.com/containerd/ttrpc.defaultClientInterceptor(0x22bf240, 0xc0008909c0, 0xc0005dc360, 0xc0000dd000, 0xc0009e9110, 0xc0009e9120, 0x1e, 0x2030002)
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:49 +0x55
github.com/containerd/ttrpc.(*Client).Call(0xc000440580, 0x22bf240, 0xc0008909c0, 0x20bc79a, 0x17, 0x20a1a36, 0x5, 0x2009c80, 0xc000890b70, 0x2009dc0, ...)
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:132 +0x33a
github.com/containerd/containerd/runtime/v2/task.(*taskClient).Stats(0xc000006e78, 0x22bf240, 0xc0008909c0, 0xc000890b70, 0xf, 0xd6c3ad, 0x6109a3fb)
D:/a/containerd/containerd/src/github.com/containerd/containerd/runtime/v2/task/shim.pb.go:3697 +0xc5
github.com/containerd/containerd/runtime/v2.(*shim).Stats(0xc0002449c0, 0x22bf240, 0xc0008909c0, 0x1, 0x1, 0xc00008cd30)
D:/a/containerd/containerd/src/github.com/containerd/containerd/runtime/v2/shim.go:460 +0xa7
github.com/containerd/containerd/services/tasks.getTasksMetrics(0x22bf240, 0xc0008909c0, 0x2290740, 0xc0005d8330, 0xc000e21400, 0x1e, 0x20, 0xc0006596c0)
D:/a/containerd/containerd/src/github.com/containerd/containerd/services/tasks/local.go:623 +0x1aa
github.com/containerd/containerd/services/tasks.(*local).Metrics(0xc0003907d0, 0x22bf240, 0xc0008909c0, 0xc0006590c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc000c45400)
D:/a/containerd/containerd/src/github.com/containerd/containerd/services/tasks/local.go:580 +0x17b
github.com/containerd/containerd/pkg/cri/server.(*criService).ListContainerStats(0xc00043d500, 0x22bf240, 0xc0008909c0, 0xc0009e8db0, 0x22bf240, 0xc0008909c0, 0x1)
D:/a/containerd/containerd/src/github.com/containerd/containerd/pkg/cri/server/container_stats_list.go:38 +0x1b3
github.com/containerd/containerd/pkg/cri/server.(*instrumentedService).ListContainerStats(0xc00008c238, 0x22bf240, 0xc000890780, 0xc0009e8db0, 0x0, 0x0, 0x0)
D:/a/containerd/containerd/src/github.com/containerd/containerd/pkg/cri/server/instrumented_service.go:422 +0x1c9
k8s.io/cri-api/pkg/apis/runtime/v1alpha2._RuntimeService_ListContainerStats_Handler.func1(0x22bf240, 0xc000890780, 0x201df00, 0xc0009e8db0, 0x33, 0xc000c29310, 0xc00077fc00, 0x0)
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.pb.go:8218 +0x8d
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1(0x22bf240, 0xc000890780, 0x201df00, 0xc0009e8db0, 0xc0000dcee0, 0xc0005d81b0, 0xc000d97b80, 0xd75e56, 0x1fc0cc0, 0xc000890780)
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0xb8
k8s.io/cri-api/pkg/apis/runtime/v1alpha2._RuntimeService_ListContainerStats_Handler(0x2073b20, 0xc00008c238, 0x22bf240, 0xc000890780, 0xc0005dc060, 0xc00049d2b0, 0x22bf240, 0xc000890780, 0xc000a149b0, 0x2)
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.pb.go:8220 +0x15e
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000084600, 0x22d07f8, 0xc0003ce900, 0xc0002e2700, 0xc000424240, 0x1dad508, 0x0, 0x0, 0x0)
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:1024 +0x522
google.golang.org/grpc.(*Server).handleStream(0xc000084600, 0x22d07f8, 0xc0003ce900, 0xc0002e2700, 0x0)
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:1313 +0xd4c
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc00014e4c0, 0xc000084600, 0x22d07f8, 0xc0003ce900, 0xc0002e2700)
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:722 +0xb2
created by google.golang.org/grpc.(*Server).serveStreams.func1
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:720 +0xa8
-----------------------------------------------------------------------------------------
ctr pprof goroutines | sls -Pattern pollwait -Context 0, 30
> internal/poll.runtime_pollWait(0x1ed32f0c900, 0x72, 0x2291780)
C:/hostedtoolcache/windows/go/1.16.6/x64/src/runtime/netpoll.go:222 +0x65
internal/poll.(*pollDesc).wait(0xc0002dc938, 0x72, 0x1d39a00, 0x0, 0x0)
C:/hostedtoolcache/windows/go/1.16.6/x64/src/internal/poll/fd_poll_runtime.go:87 +0x4c
internal/poll.execIO(0xc0002dc798, 0xc000747b28, 0xf0, 0xc0003385a0, 0x20)
C:/hostedtoolcache/windows/go/1.16.6/x64/src/internal/poll/fd_windows.go:175 +0x113
internal/poll.(*FD).acceptOne(0xc0002dc780, 0x218, 0xc0003385a0, 0x2, 0x2, 0xc0002dc798, 0xc0005d65b0, 0x1ed0d920598, 0xc000080800, 0x0)
C:/hostedtoolcache/windows/go/1.16.6/x64/src/internal/poll/fd_windows.go:810 +0x9c
internal/poll.(*FD).Accept(0xc0002dc780, 0xc000747cd8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
C:/hostedtoolcache/windows/go/1.16.6/x64/src/internal/poll/fd_windows.go:844 +0x168
net.(*netFD).accept(0xc0002dc780, 0x30, 0x30, 0x1ed0d920598)
C:/hostedtoolcache/windows/go/1.16.6/x64/src/net/fd_windows.go:139 +0x85
net.(*TCPListener).accept(0xc00038c5a0, 0xc000747d70, 0xcd027f, 0x30)
C:/hostedtoolcache/windows/go/1.16.6/x64/src/net/tcpsock_posix.go:139 +0x39
net.(*TCPListener).Accept(0xc00038c5a0, 0x1fc0cc0, 0xc0002eb110, 0x1ebebe0, 0x1d9d000)
C:/hostedtoolcache/windows/go/1.16.6/x64/src/net/tcpsock.go:261 +0x6b
net/http.(*Server).Serve(0xc000264540, 0x22b9080, 0xc00038c5a0, 0x0, 0x0)
C:/hostedtoolcache/windows/go/1.16.6/x64/src/net/http/server.go:2961 +0x29c
github.com/containerd/containerd/pkg/cri/streaming.(*server).Start(0xc00029a240, 0xc000747f01, 0xd4573b, 0xc0002eaae8)
D:/a/containerd/containerd/src/github.com/containerd/containerd/pkg/cri/streaming/server.go:265 +0x158
github.com/containerd/containerd/pkg/cri/server.(*criService).Run.func2(0xc0000cb440, 0xc00043d500)
D:/a/containerd/containerd/src/github.com/containerd/containerd/pkg/cri/server/service.go:210 +0x86
created by github.com/containerd/containerd/pkg/cri/server.(*criService).Run
D:/a/containerd/containerd/src/github.com/containerd/containerd/pkg/cri/server/service.go:208 +0x530
goroutine 2262 [select, 11 minutes]:
github.com/containerd/ttrpc.(*Client).run(0xc0000a4b00)
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:271 +0x325
created by github.com/containerd/ttrpc.NewClient
D:/a/containerd/containerd/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:94 +0x1cc
Raw
kubelet
goroutine 36734 [select, 1 minutes]:
k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002277a00)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport/transport.go:318 +0xa5
k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/internal/transport/transport.go:333
k8s.io/kubernetes/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc0025f2580, 0x56ea8a0, 0xc00088eba0, 0x0, 0x0, 0x0)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:871 +0x765
k8s.io/kubernetes/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0025f2580, 0xc00221c478, 0x2)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:736 +0x4d
k8s.io/kubernetes/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc0027679e0, 0xc0019c03c8, 0xc0019c0398, 0xc00221c47a, 0xc002d09b90)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:594 +0xa6
k8s.io/kubernetes/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc0027679e0, 0x56ea8a0, 0xc00088eba0, 0x0, 0x0)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/stream.go:735 +0x10d
k8s.io/kubernetes/vendor/google.golang.org/grpc.invoke(0x5fce8f0, 0xc0008f9f80, 0x59be4c7, 0x33, 0x56ea760, 0xc000f18f70, 0x56ea8a0, 0xc00088eba0, 0xc000236a80, 0xc0002839c0, ...)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/call.go:73 +0x14f
k8s.io/kubernetes/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc000236a80, 0x5fce8f0, 0xc0008f9f80, 0x59be4c7, 0x33, 0x56ea760, 0xc000f18f70, 0x56ea8a0, 0xc00088eba0, 0x0, ...)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/google.golang.org/grpc/call.go:37 +0x1c5
k8s.io/kubernetes/vendor/k8s.io/cri-api/pkg/apis/runtime/v1alpha2.(*runtimeServiceClient).ListContainerStats(0xc000007768, 0x5fce8f0, 0xc0008f9f80, 0xc000f18f70, 0x0, 0x0, 0x0, 0xc00274a228, 0xc00274a1a0, 0xc002346380)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.pb.go:7690 +0xd6
k8s.io/kubernetes/pkg/kubelet/cri/remote.(*remoteRuntimeService).ListContainerStats(0xc00088e300, 0xc002d09b60, 0x0, 0x0, 0x0, 0x0, 0x0)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/kubelet/cri/remote/remote_runtime.go:560 +0x1fb
k8s.io/kubernetes/pkg/kubelet/stats.(*criStatsProvider).ListPodCPUAndMemoryStats(0xc0008818f0, 0x1, 0x0, 0xc002ad5600, 0x0, 0x0)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/kubelet/stats/cri_stats_provider.go:239 +0x348
k8s.io/kubernetes/pkg/kubelet.(*Kubelet).ListPodCPUAndMemoryStats(0xc0008fe000, 0x58e5a7d, 0x1, 0x0, 0xc002ad5600, 0x0)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/kubelet/kubelet.go:1190 +0x42
k8s.io/kubernetes/pkg/kubelet/server/stats.(*summaryProviderImpl).GetCPUAndMemoryStats(0xc0009bec80, 0xc002a1e898, 0xc00264e160, 0xc002a1e890)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/kubelet/server/stats/summary.go:133 +0x25d
k8s.io/kubernetes/pkg/kubelet/server/stats.(*handler).handleSummary(0xc00088f740, 0xc002ae1440, 0xc0026f2b60)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/kubelet/server/stats/handler.go:152 +0x15e
k8s.io/kubernetes/vendor/github.com/emicklei/go-restful.(*Container).dispatch.func5(0xc002ae1440, 0xc0026f2b60)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/emicklei/go-restful/container.go:283 +0x45
k8s.io/kubernetes/vendor/github.com/emicklei/go-restful.(*FilterChain).ProcessFilter(0xc002ae14d0, 0xc002ae1440, 0xc0026f2b60)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/emicklei/go-restful/filter.go:21 +0x92
k8s.io/kubernetes/pkg/kubelet/server.(*Server).InstallAuthFilter.func1(0xc002ae1440, 0xc0026f2b60, 0xc002ae14d0)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/kubelet/server/server.go:297 +0x786
k8s.io/kubernetes/vendor/github.com/emicklei/go-restful.(*FilterChain).ProcessFilter(0xc002ae14d0, 0xc002ae1440, 0xc0026f2b60)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/emicklei/go-restful/filter.go:19 +0x6c
k8s.io/kubernetes/vendor/github.com/emicklei/go-restful.(*Container).dispatch(0xc0008e75f0, 0x5fc9338, 0xc0026f2af0, 0xc002a1e800)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/emicklei/go-restful/container.go:285 +0x87b
net/http.HandlerFunc.ServeHTTP(0xc000b46c80, 0x5fc9338, 0xc0026f2af0, 0xc002a1e800)
/usr/local/go/src/net/http/server.go:2069 +0x4b
net/http.(*ServeMux).ServeHTTP(0xc000b1af80, 0x5fc9338, 0xc0026f2af0, 0xc002a1e800)
/usr/local/go/src/net/http/server.go:2448 +0x1b7
k8s.io/kubernetes/vendor/github.com/emicklei/go-restful.(*Container).ServeHTTP(0xc0008e75f0, 0x5fc9338, 0xc0026f2af0, 0xc002a1e800)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/emicklei/go-restful/container.go:303 +0x54
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/httplog.WithLogging.func1(0x5fcbd38, 0xc001f407c0, 0xc002277400)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/httplog/httplog.go:91 +0x35e
net/http.HandlerFunc.ServeHTTP(0xc001c1d7e0, 0x5fcbd38, 0xc001f407c0, 0xc002277400)
/usr/local/go/src/net/http/server.go:2069 +0x4b
k8s.io/kubernetes/pkg/kubelet/server.(*Server).ServeHTTP(0xc000b485a0, 0x5fcbd38, 0xc001f407c0, 0xc002277400)
/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/kubelet/server/server.go:939 +0x61c
net/http.serverHandler.ServeHTTP(0xc0001fb340, 0x5fcbd38, 0xc001f407c0, 0xc002277400)
/usr/local/go/src/net/http/server.go:2887 +0xaa
net/http.initALPNRequest.ServeHTTP(0x5fce998, 0xc0006328d0, 0xc00119a380, 0xc0001fb340, 0x5fcbd38, 0xc001f407c0, 0xc002277400)
/usr/local/go/src/net/http/server.go:3459 +0xa5
net/http.(*http2serverConn).runHandler(0xc0012fa300, 0xc001f407c0, 0xc002277400, 0xc0002969a8)
/usr/local/go/src/net/http/h2_bundle.go:5723 +0x92
created by net/http.(*http2serverConn).processHeaders
/usr/local/go/src/net/http/h2_bundle.go:5453 +0x505
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment