Failure to get metrics

readme.md

steps to reproduce:

Deploy pod and create cpu spike

k apply -f deployment.yaml
k exec -it iis-deployment-pod powershell
$result = 1; foreach ($number in 1..2147483647) {$result = $result * $number};

from another machine

k proxy

for (( ; ; ))
do
   curl http://localhost:8001/api/v1/nodes/1241k8s00000000/proxy/stats/summary
done

log onto windows node:

cat c:\k\kubelet.err.log | Select-String "/stats/summary"

E0930 20:21:30.902654 5020 handler.go:321] HTTP InternalServerError serving /stats/summary: Internal Error: failed to list pod stats: failed to list all container
stats: rpc error: code = Unknown desc = container 46527a779618ed4197babc79889886b540f352d72503656eda3c37c2f247c4a0 encountered an error during Properties: failure in a
Windows system call: The requested virtual machine or container operation is not valid in the current state. (0xc0370105) E0930 20:21:30.983515 5020 handler.go:321] HTTP InternalServerError serving /stats/summary: Internal Error: failed to list pod stats: failed to list all container
stats: rpc error: code = Unknown desc = container 46527a779618ed4197babc79889886b540f352d72503656eda3c37c2f247c4a0 encountered an error during Properties: failure in a
Windows system call: The requested virtual machine or container operation is not valid in the current state. (0xc0370105) E0930 20:21:31.079794 5020 handler.go:321] HTTP InternalServerError serving /stats/summary: Internal Error: failed to list pod stats: failed to list all container
stats: rpc error: code = Unknown desc = container 46527a779618ed4197babc79889886b540f352d72503656eda3c37c2f247c4a0 encountered an error during Properties: failure in a
Windows system call: The requested virtual machine or container operation is not valid in the current state. (0xc0370105) E0930 20:21:31.145063 5020 handler.go:321] HTTP InternalServerError serving /stats/summary: Internal Error: failed to list pod stats: failed to list all container
stats: rpc error: code = Unknown desc = container 46527a779618ed4197babc79889886b540f352d72503656eda3c37c2f247c4a0 encountered an error during Properties: failure in a
Windows system call: The requested virtual machine or container operation is not valid in the current state. (0xc0370105) E0930 20:21:31.218007 5020 handler.go:321] HTTP InternalServerError serving /stats/summary: Internal Error: failed to list pod stats: failed to list all container
stats: rpc error: code = Unknown desc = container 46527a779618ed4197babc79889886b540f352d72503656eda3c37c2f247c4a0 encountered an error during Properties: failure in a
Windows system call: The requested virtual machine or container operation is not valid in the current state. (0xc0370105) E0930 20:21:31.284007 5020 handler.go:321] HTTP InternalServerError serving /stats/summary: Internal Error: failed to list pod stats: failed to list all container
stats: rpc error: code = Unknown desc = container 46527a779618ed4197babc79889886b540f352d72503656eda3c37c2f247c4a0 encountered an error during Properties: failure in a
Windows system call: The requested virtual machine or container operation is not valid in the current state. (0xc0370105) E0930 20:21:34.026254 5020 handler.go:321] HTTP InternalServerError serving /stats/summary: Internal Error: failed to list pod stats: failed to list all container
stats: rpc error: code = Unknown desc = container d8a566cef3d9ed97442c37ee5f26fb4ff51aac19ad9c1b4bb6caf84acaf0afa5 encountered an error during Properties: failure in a
Windows system call: Access is denied. (0x5)

k top nodes
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
1241k8s00000000 1556m 77% 776Mi 12%
k8s-agentlinux-12414425-vmss000000 72m 3% 830Mi 11%
k8s-master-12414425-0 370m 18% 1464Mi 20%

k top pods NAME CPU(cores) MEMORY(bytes)
iis-1809-65f5948768-4hzgp 958m 97Mi
test 0m 27Mi

deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: iis-1809
  labels:
    app: iis-1809
spec:
  replicas: 1
  template:
    metadata:
      name: iis-1809
      labels:
        app: iis-1809
    spec:
      containers:
      - name: iis
        image: mcr.microsoft.com/windows/servercore/iis:windowsservercore-ltsc2019
        resources:
          limits:
            cpu: 1
            memory: 800m
          requests:
            cpu: .1
            memory: 300m
        ports:
          - containerPort: 80
      nodeSelector:
        "kubernetes.io/os": windows
      tolerations:
        - key: ""
          operator: "Exists"
          effect: "NoSchedule"
  selector:
    matchLabels:
      app: iis-1809
---
apiVersion: v1
kind: Service
metadata:
  name: iis
spec:
  type: LoadBalancer
  ports:
  - protocol: TCP
    port: 80
  selector:
    app: iis-1809

model.json

{
	"apiVersion": "vlabs",
	"properties": {
		"orchestratorProfile": {
			"orchestratorType": "Kubernetes",
			"orchestratorRelease": "1.18",
			"orchestratorVersion": "1.18.8"
		},
		"masterProfile": {
			"count": 1,
			"dnsPrefix": "",
			"vmSize": "Standard_D2s_v3"
		},
		"agentPoolProfiles": [
			{
				"name": "agentwin",
				"count": 1,
				"vmSize": "Standard_D2s_v3",
				"osType": "Windows"
			},
			{
				"name": "agentlinux",
				"count": 1,
				"vmSize": "Standard_D2_v3"
			}
		],
		"windowsProfile": {
			"adminUsername": "azureuser",
			"adminPassword": "replacepassword1234$",
			"enableAutomaticUpdates": false,
			"sshEnabled": true,
			"windowsPublisher": "microsoft-aks",
			"windowsOffer": "aks-windows",
            "windowsSku": "2019-datacenter-core-smalldisk-2008",
            "imageVersion": "latest"
		},
		"linuxProfile": {
			"adminUsername": "azureuser",
			"ssh": {
				"publicKeys": [
					{
						"keyData": ""
					}
				]
			}
		},
		"servicePrincipalProfile": {
			"clientId": "",
			"secret": ""
		}
	}
}