thxCode · June 2, 2021 13:21
diff --git a/windows-troubleshooting-summary.md b/windows-troubleshooting-summary.md
diff --git a/zz_254_windows_pods_delete.sh b/zz_254_windows_pods_delete.sh
 #!/bin/sh

 yaml=$(curl https://gist.githubusercontent.com/thxCode/f0c6964d05d2149f400dfb13a3cf0607/raw/9484155eedfb9dabfb319ada32a3f724b2ce00cd/zz_cluster_connect.yaml)

 tmpfile=$(mktemp)

 echo "delete ..."
 for i in $(seq 1 254); do
  echo "${yaml}" | sed "s# win-openresty# win-openresty-${i}#g" >${tmpfile}
  kubectl delete -f ${tmpfile}
 done
 echo "finished"
diff --git a/zz_256_windows_pods_apply.sh b/zz_256_windows_pods_apply.sh
 #!/bin/sh

 yaml=$(curl https://gist.githubusercontent.com/thxCode/f0c6964d05d2149f400dfb13a3cf0607/raw/9484155eedfb9dabfb319ada32a3f724b2ce00cd/zz_cluster_connect.yaml)

 tmpfile=$(mktemp)

 echo "warm up ..."
 for i in $(seq 1 10); do
 echo "${yaml}" | sed "s# win-openresty# win-openresty-${i}#g" >${tmpfile}
 kubectl apply -f ${tmpfile}
 done

 sleep 60s

 echo "create ..."
 for i in $(seq 10 256); do
 echo "${yaml}" | sed "s# win-openresty# win-openresty-${i}#g" >${tmpfile}
 kubectl apply -f ${tmpfile}
 sleep 30s
 done

 echo "finished"
diff --git a/zz_cluster_connect.ps1 b/zz_cluster_connect.ps1
 function Log-Info
 {
    Write-Host -NoNewline -ForegroundColor Blue "INFO "
    Write-Host -ForegroundColor Gray ("{0,-44}" -f ($args -join " "))
 }

 function Log-Warn
 {
    Write-Host -NoNewline -ForegroundColor DarkYellow "WARN "
    Write-Host -ForegroundColor Gray ("{0,-44}" -f ($args -join " "))
 }

 function Log-Error
 {
    Write-Host -NoNewline -ForegroundColor DarkRed "ERRO "
    Write-Host -ForegroundColor Gray ("{0,-44}" -f ($args -join " "))
 }

 function Log-Fatal
 {
    Write-Host -NoNewline -ForegroundColor DarkRed "FATA "
    Write-Host -ForegroundColor Gray ("{0,-44}" -f ($args -join " "))

    exit 255
 }

 function Wait
 {
    param(
        [parameter(Mandatory = $true, ValueFromPipeline = $true)] [scriptBlock]$Block,
        [parameter(Mandatory = $false)] [int]$Timeout = 30,
        [parameter(Mandatory = $false)] [switch]$Reverse,
        [parameter(Mandatory = $false)] [string]$Throw
    )

    $count = $Timeout
    while ($count -gt 0) {
        Start-Sleep -s 1

        if (&$Block) {
            if (-not $Reverse) {
                Start-Sleep -s 5
                break
            }
        } elseif ($Reverse) {
            Start-Sleep -s 5
            break
        }

        Start-Sleep -s 1
        $count -= 1
    }

    if ($count -le 0) {
        if ($Throw) {
            throw $Throw
        }

        Log-Fatal "Timeout"
        return $False
    }
    return $True
 }

 function Get-ClusterJson
 {
    param
    (
        [parameter(Mandatory = $true)] [string] $Token,
        [parameter(Mandatory = $true)] [string] $URL
    )

    $resp = $(curl.exe -skL -H 'Accept: application/json' -H 'Content-Type: application/json' -H "Authorization: Bearer $Token" -w "%{http_code}" "$URL")
    if (-not $?) {
        throw "Failed to get cluster json : $resp"
    }

    $respCode = $resp | Select-Object -Last 1
    $respContent = $resp | Select-Object -First 1
    if ($respCode -ne 200) {
        Log-Fatal "Failed to get cluster json : ($respCode) $respContent"
    }

    return $respContent
 }

 function Get-Kubecfg
 {
    param
    (
        [parameter(Mandatory = $true)] [string] $Token,
        [parameter(Mandatory = $true)] [string] $ClusterJson
    )

    $clusterJsonObj = $ClusterJson | ConvertFrom-Json
    $generateKubeconfigURL = $clusterJsonObj.data[0].actions.generateKubeconfig

    $resp = $(curl.exe -skL -X POST -H 'Accept: application/json' -H 'Content-Type: application/json' -d '{}' -H "Authorization: Bearer $Token" -w "%{http_code}" "$generateKubeconfigURL")
    if (-not $?) {
        throw "Failed to get kubecfg : $resp"
    }
    $respCode = $resp | Select-Object -Last 1
    $respContent = $resp | Select-Object -First 1
    if ($respCode -ne 200) {
        throw "Failed to get kubecfg : ($respCode) $respContent"
    }

    $respContentObj = $respContent | ConvertFrom-Json
    $respContentObj.config | Out-File -FilePath "$env:TEMP\kubecfg.yaml"
    return @{
        dir = $env:TEMP
        file = "kubecfg.yaml"
    }
 }

 function Get-MgmtIP
 {
    while ($True) {
        $hnsNetwork = Get-HnsNetwork | Where-Object {($_.Name -eq "vxlan0") -or ($_.Name -eq "cbr0")}
        if ($hnsNetwork -and $hnsNetwork.ManagementIp -and (Get-NetIPAddress $hnsNetwork.ManagementIP -ErrorAction Ignore)) {
            return $hnsNetwork.ManagementIp
        }
        Start-Sleep -s 1
    }
 }

 function Kubectl
 {
    param
    (
        [parameter(Mandatory = $true)] $Cfg,
        [parameter(Mandatory = $true)] $Arguments
    )

    $kubectlImage = $(docker inspect --format='{{.Config.Image}}' kubelet)
    $kubecfgHostDir = $Cfg.dir
    $kubecfgHostFile = $Cfg.file

    return $(docker run --rm -v "$($kubecfgHostDir):c:\host\kubecfgdir" -e "KUBECONFIG=c:\host\kubecfgdir\$kubecfgHostFile" $kubectlImage kubectl $Arguments 2>/dev-null)
 }

 if (-not $RANCHER_API_ENDPOINT) {
    if ($env:RANCHER_API_ENDPOINT) {
        $RANCHER_API_ENDPOINT = $env:RANCHER_API_ENDPOINT
    }
 }
 if (-not $RANCHER_API_ENDPOINT) {
    Log-Fatal "Please indicate `$RANCHER_API_ENDPOINT (e.g.: https://your.server.ip/v3)"
 }
 Log-Info "`$RANCHER_API_ENDPOINT=$RANCHER_API_ENDPOINT"

 if (-not $RANCHER_API_BEARER_TOKEN) {
    if ($env:RANCHER_API_BEARER_TOKEN) {
        $RANCHER_API_BEARER_TOKEN = $env:RANCHER_API_BEARER_TOKEN
    }
 }
 if (-not $RANCHER_API_BEARER_TOKEN) {
    Log-Fatal "Please indicate `$RANCHER_API_BEARER_TOKEN (e.g.: token-h9r9t:mp9xb6mm7k8s4vbz9brjlh68wt7qgrhs4hnf24lbb557lrlcpvmlgn)"
 }
 Log-Info "`$RANCHER_API_BEARER_TOKEN=$($RANCHER_API_BEARER_TOKEN.substring(0, 12))*******"

 if (-not $RANCHER_CLUSTER_ID) {
    if ($env:RANCHER_CLUSTER_ID) {
        $RANCHER_CLUSTER_ID = $env:RANCHER_CLUSTER_ID
    }
 }
 if (-not $RANCHER_CLUSTER_NAME) {
    if ($env:RANCHER_CLUSTER_NAME) {
        $RANCHER_CLUSTER_NAME = $env:RANCHER_CLUSTER_NAME
    }
 }
 if ((-not $RANCHER_CLUSTER_NAME) -and (-not $RANCHER_CLUSTER_ID)) {
    Log-Fatal "Please indicate `$RANCHER_CLUSTER_NAME (e.g.: windows-cluster) or `$RANCHER_CLUSTER_ID (e.g.: c-xwfgv)"
 }

 # get kubecfg
 $kubecfg = @{}
 if ($RANCHER_CLUSTER_NAME) {
    Log-Info "`$RANCHER_CLUSTER_NAME=$RANCHER_CLUSTER_NAME"
    try {
        $clusterJson = Get-ClusterJson -Token $RANCHER_API_BEARER_TOKEN -URL "$RANCHER_API_ENDPOINT/clusters/?name=$RANCHER_CLUSTER_NAME"
        $kubecfg = Get-Kubecfg -Token $RANCHER_API_BEARER_TOKEN -ClusterJson $clusterJson
    } catch {
        Log-Fatal "Failed to gain kubecfg of cluster via `$RANCHER_CLUSTER_NAME=$RANCHER_CLUSTER_NAME : $($_.Exception.Messsage)"
    }
 } else {
    Log-Info "`$RANCHER_CLUSTER_ID=$RANCHER_CLUSTER_ID"
    try {
        $clusterJson = Get-ClusterJson -Token $RANCHER_API_BEARER_TOKEN -URL "$RANCHER_API_ENDPOINT/clusters/?id=$RANCHER_CLUSTER_ID"
        $kubecfg = Get-Kubecfg -Token $RANCHER_API_BEARER_TOKEN -ClusterJson $clusterJson
    } catch {
        Log-Fatal "Failed to gain kubecfg of cluster via `$RANCHER_CLUSTER_ID=$RANCHER_CLUSTER_ID : $($_.Exception.Messsage)"
    }
 }
 Log-Info "Got kubecfg from Rancher server"

 Describe 'Basic Connectivity Tests' {
    BeforeAll {
        $workloadName = "win-openresty"

        # kubectl apply
        Kubectl -Cfg $kubecfg -Arguments @(
            "apply", 
            "-f", 
            "https://gist.githubusercontent.com/thxCode/f0c6964d05d2149f400dfb13a3cf0607/raw/9484155eedfb9dabfb319ada32a3f724b2ce00cd/zz_cluster_connect.yaml"
        )
        Log-Info "Applied testing Workload"

        # scale the workload
        $replicas = Kubectl -Cfg $kubecfg -Arguments @(
            "get", 
            "nodes", 
            "-o",
            "jsonpath={range .items[?(@.status.nodeInfo.operatingSystem=='windows')]}{.metadata.name}{'\n'}{end}"
        ) | Measure-Object | Select-Object -ExpandProperty Count
        $replicas = $replicas + 2
        Kubectl -Cfg $kubecfg -Arguments @(
            "scale", 
            "deployment", 
            $workloadName,
            "--replicas=$replicas"
        )
         Log-Info "Scaled testing Worload to $replicas"

        # get windows nodes
        {
            $workload = Kubectl -Cfg $kubecfg -Arguments @(
                "get", 
                "deployment", 
                $workloadName,
                "-o",
                "json"
            ) | ConvertFrom-Json
            if ($workload -and ($workload.status.availableReplicas -eq $workload.status.replicas)) {
                return $True
            }
            return $False
        } | Wait -Timeout 3000 -Throw "Could not deploy Deployment($workloadName)"
        Log-Info "Testing Workload got ready"

        $service = Kubectl -Cfg $kubecfg -Arguments @(
                    "get", 
                    "service",
                    $workloadName,
                    "-o",
                    "json"
                ) | ConvertFrom-Json | ForEach-Object {
                    return @{
                        name = $workloadName
                        ip = $_.spec.clusterIP
                        nodePort = $_.spec.ports[0].nodePort
                    }
                }
        $pods = Kubectl -Cfg $kubecfg -Arguments @(
                    "get", 
                    "pods",
                    "-o",
                    "name"
                ) | Select-String $workloadName | ForEach-Object {
                    $pod = Kubectl -Cfg $kubecfg -Arguments @(
                       "get", 
                        $_,
                        "-o",
                        "json"
                    ) | ConvertFrom-Json

                    return @{
                        name = $_
                        ip = $pod.status.podIP
                        hostIP = $pod.status.hostIP
                    }
                }
        $managementIP = Get-MgmtIP
        $localPods = $pods | Where-Object {$_.hostIP -eq $managementIP}
        $remotePods = $pods | Where-Object {$_.hostIP -ne $managementIP}
    }

    AfterAll {
        # kubectl delete
        Kubectl -Cfg $kubecfg -Arguments @(
            "delete", 
            "-f", 
            "https://gist.githubusercontent.com/thxCode/f0c6964d05d2149f400dfb13a3cf0607/raw/6e92f294093c968284a1f3e2ae29040a187ab4fc/zz_cluster_connect.yaml"
        )
        Log-Info "Deleted testing Workload"
    }

    It 'Should Pods have correct IP' {
        $pods.Length | Should Be $replicas
        $pods | ForEach-Object {
            # ipconfig match test
            Kubectl -Cfg $kubecfg -Arguments @(
                "exec", 
                $_.name,
                "--",
                "ipconfig"
            ) | Out-String | Should Match $_.ip
        }
    }

    It 'Should Pods have Internet connectivity' {
        $pods.Length | Should Be $replicas
        $pods | ForEach-Object {
            # curl test
            Kubectl -Cfg $kubecfg -Arguments @(
                "exec", 
                $_.name,
                "--",
                "curl.exe",
                "-skL",
                "-w",
                '"%{http_code}"',
                "-o",
                "/dev-null",
                "https://www.google.com"
            ) | Out-String | Should Be 200
        }
    }

    It 'Should Pods be able to resolve same namespace Service Name' {
        $pods.Length | Should Be $replicas
        $pods | ForEach-Object {
            # curl test
            Kubectl -Cfg $kubecfg -Arguments @(
                "exec", 
                $_.name,
                "--",
                "curl.exe",
                "-sL",
                "-w",
                '"%{http_code}"',
                "-o",
                "/dev-null",
                "http://$workloadName"
            ) | Out-String | Should Be 200
        }
    }

    It 'Should Host be able to reach Service IP' {
        $service | Should Not BeNullOrEmpty
        # curl test
        curl.exe -sL -w "%{http_code}" -o /dev-null "http://$($service.ip)" | Should Be 200
    }

    It 'Should local Pods be able to reach local Host' {
        $localPods.Length | Should BeGreaterThan 0
        $localPods | ForEach-Object {
            # ping test
            $pingResult = Kubectl -Cfg $kubecfg -Arguments @(
                "exec", 
                $_.name,
                "--",
                "ping",
                $_.hostIP,
                "-n",
                "4"
            ) | Out-String

            if (-not ($pingResult -match "\(0% .*\)")) {
                Log-Warn "Please enable the ICMPv4-In firewall rule, e.g.: Enable-NetFirewallRule -DisplayName 'File and Printer Sharing (Echo Request - ICMPv4-In)'"
            }

            $pingResult | Should Match "\(0% .*\)"
        }
    }

    It 'Should Host be able to reach local Pods' {
        $localPods.Length | Should BeGreaterThan 0
        $localPods | ForEach-Object {
            # ping test
            ping $_.ip -n 4 | Out-String | Should Match "\(0% .*\)"
        }
    }

    It 'Should local Pod be able to ping each other' {
        $localPods.Length | Should BeGreaterThan 0
        $localPods | ForEach-Object {
            $src = $_
            $localPods | Where-Object {$_.name -ne $src.name} | ForEach-Object {
                $dst = $_
                # ping test
                Kubectl -Cfg $kubecfg -Arguments @(
                    "exec", 
                    $src.name,
                    "--",
                    "ping",
                    $dst.ip,
                    "-n",
                    "4"
                ) | Out-String | Should Match "\(0% .*\)"
            }
        }
    } 

    It 'Should local Pod be able to ping each remote Pod' {
        $localPods.Length | Should BeGreaterThan 0
        if ($remotePods.Length -eq 0) {
            Set-TestInconclusive -Message "There aren't any remote Pods"
        }

        $localPods | ForEach-Object {
            $src = $_
            $remotePods | ForEach-Object {
                $dst = $_
                # ping test
                Kubectl -Cfg $kubecfg -Arguments @(
                    "exec", 
                    $src.name,
                    "--",
                    "ping",
                    $dst.ip,
                    "-n",
                    "4"
                ) | Out-String | Should Match "\(0% .*\)"
            }
        }
    } 

    It 'Should Host be able to access Node port' {
        $service | Should Not BeNullOrEmpty

        if ($remotePods.Length -eq 0) {
            Set-TestInconclusive -Message "There aren't any remote Pods"
        }

        $remotePods | ForEach-Object {
            # curl test
            curl.exe -sL -w "%{http_code}" -o /dev-null "http://$($_.hostIP):$($service.nodePort)" | Should Be 200
        }
    }
 }
diff --git a/zz_cluster_connect.yaml b/zz_cluster_connect.yaml
 apiVersion: v1
 kind: Service
 metadata:
  name: win-openresty
  labels:
    app: win-openresty
 spec:
  ports:
  - port: 80
    targetPort: 80
  selector:
    app: win-openresty
  type: NodePort
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  labels:
    app: win-openresty
  name: win-openresty
 spec:
  selector:
    matchLabels:
      app: win-openresty
  template:
    metadata:
      labels:
        app: win-openresty
      name: win-openresty
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: beta.kubernetes.io/os
                operator: In
                values:
                - windows
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - windows
      containers:
      - name: windowswebserver
        image: maiwj/win-openresty:windows
diff --git a/zz_k8s_run.ps1 b/zz_k8s_run.ps1
 Describe "Windows version and Prerequisites" {
    $osVersion = Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion'

    It "Is Windows Server" {
        (Get-ComputerInfo).WindowsInstallationType | Should MatchExactly '^(Server|Server Core)$'
    }

    It "Has 'Containers' feature installed" {
        (Get-WindowsFeature -Name Containers).InstallState | Should MatchExactly '^Installed$'
    }

    It "Should be a supported version" {
        $osVersion.CurrentBuildNumber -ge 17763 | Should Be $true
    }

    It "Could support 'VxLAN' network (has KB4489899 installed)" {
        if ($osVersion.CurrentBuildNumber -eq 17763) {
            $osVersion.UBR | Should Not BeLessThan 379
        }
    }

    It "Is the recommanded Windows UBR version" {
        if ($osVersion.CurrentBuildNumber -eq 17763) {
            (($osVersion.UBR -ge 404) -and ($osVersion.UBR -le 504)) | Should Be $false
        }
    }

    Write-Host "===OS version==="
    Write-Host $($osVersion | Format-List -Property CurrentMajorVersionNumber,CurrentMinorVersionNumber,CurrentBuildNumber,UBR,ReleaseId,BuildLabEx,CurrentBuild | Out-String)
    Write-Host "================"
 }

 Describe "Docker version and Prerequisites" {
    $dockerService = Get-Service -Name Docker -ErrorAction Ignore

    It "Has Docker installed" {
        $dockerService | Should Not BeNullOrEmpty
    }

    It "Has running" {
        if (-not $dockerService) {
            Set-TestInconclusive -Message 'Could not find this service'
        }

        $dockerService.Status | Should Be 'Running'
    }

    It "Has permissions to use" {
        if (-not $dockerService) {
            Set-TestInconclusive -Message 'Could not find this service'
        } elseif ($dockerService.Status -ne 'Running') {
            Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
        }

        $stdout = New-TemporaryFile
        $stderr = New-TemporaryFile
        {
            Start-Process -NoNewWindow -Wait `
              -FilePath docker.exe `
              -ArgumentList "info" `
              -RedirectStandardOutput $stdout.FullName `
              -RedirectStandardError $stderr.FullName
        } | Should Not Throw
        $stdout.FullName | Should Not Contain "access is denied"
        $stderr.FullName | Should Not Contain "access is denied"
        $stdout.Delete()
        $stderr.Delete()
    }

    It "Has registered into EventLog" {
        if (-not $dockerService) {
            Set-TestInconclusive -Message 'Could not find this service'
        }

        Test-Path "HKLM:\SYSTEM\CurrentControlSet\Services\EventLog\Application\docker" | Should Be $true
    }

    It "Should start automatically" {
        if (-not $dockerService) {
            Set-TestInconclusive -Message 'Could not find this service'
        }

        $dockerService.StartType | Should Be 'Automatic'
    }

    It "Should be an enterprise version" {
        if (-not $dockerService) {
            Set-TestInconclusive -Message 'Could not find this service'
        } elseif ($dockerService.Status -ne 'Running') {
            Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
        }

        $stdout = New-TemporaryFile
        $stderr = New-TemporaryFile
        {
            Start-Process -NoNewWindow -Wait `
              -FilePath docker.exe `
              -ArgumentList "version -f '{{.Server.Platform.Name}}'" `
              -RedirectStandardOutput $stdout.FullName `
              -RedirectStandardError $stderr.FullName
        } | Should Not Throw
        $stdout.FullName | Should Contain "Enterprise"
        $stdout.Delete()
        $stderr.Delete()
    }

    It "Should cache at least one image of 'mcr.microsoft.com/windows/servercore', 'mcr.microsoft.com/windows/nanoserver', 'mcr.microsoft.com/windows'" {
        if (-not $dockerService) {
            Set-TestInconclusive -Message 'Could not find this service'
        } elseif ($dockerService.Status -ne 'Running') {
            Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
        }

        $dockerImages = docker.exe image ls --format '{{json .}}' | ConvertFrom-Json -ErrorAction Ignore | ? { ($_.Repository -like 'mcr.microsoft.com/*') }

        ($dockerImages | Measure-Object).Count | Should Not BeNullOrEmpty
    }

    if ($dockerService -and ($dockerService.Status -eq "Running")) {
        Write-Host "===Docker version==="
        Write-Host -NoNewline "Server version: "; Write-Host $(docker version -f '{{.Server.Version}}')
        Write-Host -NoNewline "Client version: "; Write-Host $(docker version -f '{{.Client.Version}}')
        Write-Host "==================="
    }
 }

 Describe "Rancher Wins and Prerequisites" {
    $rancherService = Get-Service -Name rancher-wins -ErrorAction Ignore

    It "Has rancher-wins installed" {
        $rancherService | Should Not BeNullOrEmpty
    }

    It "Has running" {
        if (-not $rancherService) {
            Set-TestInconclusive -Message 'Could not find this service'
        }

        $rancherService.Status | Should Be 'Running'
    }

    It "Has registered into EventLog" {
        if (-not $rancherService) {
            Set-TestInconclusive -Message 'Could not find this service'
        }

        Test-Path "HKLM:\SYSTEM\CurrentControlSet\Services\EventLog\Application\rancher-wins" | Should Be $true
    }

    It "Should start automatically" {
        if (-not $rancherService) {
            Set-TestInconclusive -Message 'Could not find this service'
        }

        $rancherService.StartType | Should Be 'Automatic'
    }

    if ($rancherService -and ($rancherService.Status -eq "Running")) {
        Write-Host "===Rancher Wins version==="
        $stdout = New-TemporaryFile
        $stderr = New-TemporaryFile
        Start-Process -NoNewWindow -Wait `
          -FilePath c:\etc\rancher\wins.exe `
          -ArgumentList "cli app info" `
          -RedirectStandardOutput $stdout.FullName `
          -RedirectStandardError $stderr.FullName
        Write-Host $(Get-Content -Raw $stdout.FullName | ConvertFrom-Json | Format-List | Out-String)
        $stdout.Delete()
        $stderr.Delete()
        Write-Host "=========================="
    }
 }

 Describe "Kubernetes components and Prerequisites" {
    $nginxProcess = Get-Process -Name rancher-wins-nginx -ErrorAction Ignore
    $kubeletProcess = Get-Process -Name rancher-wins-kubelet -ErrorAction Ignore
    $kubeproxyProcess = Get-Process -Name rancher-wins-kube-proxy -ErrorAction Ignore
    $flanneldProcess = Get-Process -Name rancher-wins-flanneld -ErrorAction Ignore

    It "Has Nginx running" {
        $nginxProcess | Should Not BeNullOrEmpty
    }

    It "Has Kubelet running" {
        $kubeletProcess | Should Not BeNullOrEmpty
    }

    It "Has Kube-proxy running" {
        $kubeproxyProcess | Should Not BeNullOrEmpty
    }

    It "Has Flanneld running" {
        $flanneldProcess | Should Not BeNullOrEmpty
    }

    if ($nginxProcess) {
        Write-Host "===Nginx version & config==="
        Start-Process -NoNewWindow -Wait `
          -FilePath c:\etc\nginx\rancher-wins-nginx.exe `
          -ArgumentList "-v"
        Write-Host "`n$(Get-Content c:\etc\nginx\conf\nginx.conf | Out-String)"
        Write-Host "============================`n"
    }

    if ($kubeletProcess) {
        Write-Host "===Kubelet version & config==="
        Start-Process -NoNewWindow -Wait `
          -FilePath c:\etc\kubernetes\bin\rancher-wins-kubelet.exe `
          -ArgumentList "--version"
        Write-Host "`n$((Get-WmiObject -Class Win32_Process -Filter "Handle=$($kubeletProcess.ID)").CommandLine)"
        Write-Host "==============================`n"
    }

    if ($kubeproxyProcess) {
        Write-Host "===Kubeproxy version & config==="
        Start-Process -NoNewWindow -Wait `
          -FilePath c:\etc\kubernetes\bin\rancher-wins-kube-proxy.exe `
          -ArgumentList "--version"
        Write-Host "`n$((Get-WmiObject -Class Win32_Process -Filter "Handle=$($kubeproxyProcess.ID)").CommandLine)"
        Write-Host "================================`n"
    }

    if ($flanneldProcess) {
        Write-Host "===Flanneld version & config==="
        Start-Process -NoNewWindow -Wait `
          -FilePath c:\opt\bin\rancher-wins-flanneld.exe `
          -ArgumentList "-version"
        Write-Host "`n$((Get-WmiObject -Class Win32_Process -Filter "Handle=$($flanneldProcess.ID)").CommandLine)"
        Write-Host $(Get-Content c:\etc\kube-flannel\net-conf.json | Out-String)
        Write-Host "==============================="
    }
 }
diff --git a/zz_v2_2_debug.ps1 b/zz_v2_2_debug.ps1
 function format-file-size {
    param ($size)

    if     ($size -gt 1TB) {[string]::Format("{0:n2} TB", $size / 1TB)}
    elseIf ($size -gt 1GB) {[string]::Format("{0:n2} GB", $size / 1GB)}
    elseIf ($size -gt 1MB) {[string]::Format("{0:n2} MB", $size / 1MB)}
    elseIf ($size -gt 1KB) {[string]::Format("{0:n2} KB", $size / 1KB)}
    elseIf ($size -gt 0)   {[string]::Format("{0:n2} B", $size)}
    else                   {"0 B"}
 }

 $resultDir = (Resolve-Path ".").Path
 $resultDir = "$resultDir\$($(hostname).ToLower())_$((get-date).ToString('yyyyMMdd'))"

 Remove-Item -Path $resultDir -Recurse -Force -ErrorAction Ignore | Out-Null
 New-Item -Type Directory -Path $resultDir -ErrorAction Ignore | Out-Null

 Describe "Host" {
    $osVersion = Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion'

    Context "Prerequisites" {
        It "Is Windows Server" {
            (Get-ComputerInfo).WindowsInstallationType | Should Be 'Server'
        }

        It "Is 'Windows Server 2019' or 'Windows Server, version 1809'" {
            $osVersion.CurrentBuildNumber -ge 17763 | Should Be $true
        }

        It "Support 'VxLAN' network (has KB4489899 installed)" {
            $osVersion.UBR | Should Not BeLessThan 379
        }

        It "Not in the non-recommended Windows Server UBR version range: [404, 504]" {
            # https://github.com/rancher/rancher/issues/20440 reports that could not schedule more than 10 Pods on the Windows node which belongs to the UBR version range [404, 504]:

            (($osVersion.UBR -ge 404) -and ($osVersion.UBR -le 504)) | Should Be $false
        }
        
        It "Has 'Containers' feature installed" {
            (Get-WindowsFeature -Name Containers).InstallState | Should Be 'Installed'
        }
    }

    Context "Snapshot" {
        It "Get OS info" {
            {   "`n# OS info #`n" | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
                $osVersion | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
            } | Should Not Throw
        }

        It "Get system info" {
            {
                "`n# System info #`n" | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
                systeminfo | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"

            } | Should Not Throw
        }

        It "Get CPU info" {
            {
                "`n# CPU info #`n" | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
                Get-WmiObject -Class Win32_Processor | Select DeviceID,Name,NumberOfCores,NumberOfLogicalProcessors | `
                Format-List | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"

            } | Should Not Throw
        }

        It "Get disk info" {
            { 
                "`n# Disk info #`n" | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
                Get-Volume | Format-List | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"

            } | Should Not Throw
        }

        It "Get memory info" {
            { 
                "`n# Memory info #`n" | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
                New-Object -Typename PSObject -Property @{ `
                    Total=(Get-WmiObject -Class Win32_PhysicalMemory | Measure-Object -Sum -Property Capacity | % { format-file-size($_.Sum) }); `
                    Available=(Get-WmiObject -Class Win32_PerfFormattedData_PerfOS_Memory |  Measure-Object -Sum -Property AvailableBytes | % { format-file-size($_.Sum) });
                } | Format-List | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
            } | Should Not Throw
        }

        It "Get hotfix" {
            {
                "`n# Hotfix info #`n" | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
                Get-hotfix | Sort-Object InstalledOn -Descending | Format-List | Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
            } | Should Not Throw
        }
    }
 }

 Describe "Docker" {
    $dockerService = Get-Service -Name Docker -ErrorAction Ignore
    $dockerService | Select Name,Status,DisplayName,StartType | Out-File -Append -Encoding utf8 -FilePath "$resultDir\docker-info.txt"

    Context "Prerequisites" {
        It "Has Docker installed" {
            $dockerService | Should Not BeNullOrEmpty
        }

        It "Is Dockerd start automatically" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            }

            $dockerService.StartType | Should Be 'Automatic'
        }

        It "Is Dockerd registered in the EventLog service" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            }

            Test-Path "HKLM:\SYSTEM\CurrentControlSet\Services\EventLog\Application\docker" | Should Be $true
        }

        It "Is Dockerd running" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            }

            $dockerService.Status | Should Be 'Running'
        }

        It "Has permissions to use Dockerd" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            } elseif ($dockerService.Status -ne 'Running') {
                Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
            }

            $stdout = New-TemporaryFile
            $stderr = New-TemporaryFile

            {
                Start-Process -NoNewWindow -Wait `
                  -FilePath docker.exe `
                  -ArgumentList "info" `
                  -RedirectStandardOutput $stdout.FullName `
                  -RedirectStandardError $stderr.FullName
            } | Should Not Throw
            $stdout.FullName | Should Not Contain "access is denied"
            $stderr.FullName | Should Not Contain "access is denied"

            $stdout.Delete()
            $stderr.Delete()
        }

        It "Is Enterprise Docker" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            } elseif ($dockerService.Status -ne 'Running') {
                Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
            }

            $stdout = New-TemporaryFile
            $stderr = New-TemporaryFile

            {
                Start-Process -NoNewWindow -Wait `
                  -FilePath docker.exe `
                  -ArgumentList @("version", "-f", "{{.Server.Platform.Name}}") `
                  -RedirectStandardOutput $stdout.FullName `
                  -RedirectStandardError $stderr.FullName
            } | Should Not Throw
            $stdout.FullName | Should Contain "Enterprise"

            $stdout.Delete()
            $stderr.Delete()
        }

        It "Should install at least one of 'mcr.microsoft.com/windows/servercore', 'mcr.microsoft.com/windows/nanoserver', 'mcr.microsoft.com/windows' or deprecated microsoft/windowsservercore, microsoft/nanoserver" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            } elseif ($dockerService.Status -ne 'Running') {
                Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
            }

            $dockerImages = docker.exe image ls --format '{{json .}}' | ConvertFrom-Json | ? { ($_.Repository -like 'mcr.microsoft.com/*') -or ($_.Repository -like 'microsoft/*') }

            ($dockerImages | Measure-Object).Count | Should Not BeNullOrEmpty
        }
    }

    Context "Snapshot" {
        function dck {
            $stdout = New-TemporaryFile
            $stderr = New-TemporaryFile

            Start-Process -NoNewWindow -Wait `
              -FilePath docker.exe `
              -ArgumentList $args `
              -RedirectStandardOutput $stdout.FullName `
              -RedirectStandardError $stderr.FullName

            "`n# docker $($args -join " ") #`n" | Out-File -Append -Encoding utf8 -FilePath "$resultDir\docker-info.txt"
            Get-Content $stdout.FullName, $stderr.FullName | Out-File -Append -Encoding utf8 -FilePath "$resultDir\docker-info.txt"

            $stdout.Delete()
            $stderr.Delete()
        }

        It "docker info" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            } elseif ($dockerService.Status -ne 'Running') {
                Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
            }

            {
                dck info 
            } | Should Not Throw
        }

        It "docker version" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            } elseif ($dockerService.Status -ne 'Running') {
                Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
            }

            {
                dck version
            } | Should Not Throw
        }

        It "docker image ls" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            } elseif ($dockerService.Status -ne 'Running') {
                Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
            }

            {
                dck image ls
            } | Should Not Throw
        }

        It "docker ps -a" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            } elseif ($dockerService.Status -ne 'Running') {
                Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
            }

            {
                dck ps -a
            } | Should Not Throw
        }

        It "docker system df" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            } elseif ($dockerService.Status -ne 'Running') {
                Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
            }

            {
                dck system df
            } | Should Not Throw
        }
    }

    Context "Collect EventLogs" {
        $eventLogProviders = @("Docker")

        It "Export logs in last hour" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            }

            {
                Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-1))} -ErrorAction Ignore | `
                  Sort-Object ProviderName,TimeCreated,Index | `
                  Export-CSV "$resultDir\docker-lasthour-logs.csv"
            } | Should Not Throw
        }

        It "Export warning & error logs in last day" {
            if (-not $dockerService) {
                Set-TestInconclusive -Message 'Could not find Docker service'
            }

            {
                Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-24)); Level=@(3,2,1,0)} -ErrorAction Ignore | `
                  Sort-Object ProviderName,TimeCreated,Index | `
                  Export-CSV "$resultDir\docker-lastday-errlogs.csv"
            } | Should Not Throw
        }
    }
 }

 Describe "Rancher Agent" {
    $rancherService = Get-Service -Name rancher-agent -ErrorAction Ignore
    $rancherService | Select Name,Status,DisplayName,StartType | Out-File -Encoding utf8 -FilePath "$resultDir\agent-info.txt"

    Context "Prerequisites" {
        It "Has Rancher Agent installed" {
            $rancherService | Should Not BeNullOrEmpty
        }

        It "Is Rancher Agent start automatically" {
            if (-not $rancherService) {
                Set-TestInconclusive -Message 'Could not find Rancher Agent service'
            }

            $rancherService.StartType | Should Be 'Automatic'
        }
    }

    Context "Collect EventLogs" {
        $eventLogProviders = @("rancher-agent")

        It "Export logs in last week hours" {
            if (-not $rancherService) {
                Set-TestInconclusive -Message 'Could not find Rancher Agent service'
            }

            {
                Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(7 * -24))} -ErrorAction Ignore | `
                  Sort-Object ProviderName,TimeCreated,Index | `
                  Export-CSV "$resultDir\agent-lastweek-logs.csv"
            } | Should Not Throw
        }
    }
 }

 Describe "Kubernetes Components" {
    Context "Prerequisites" {
        function log-process-info {
            param (
                [parameter(Mandatory = $true)] [string]$Name,
                [parameter(Mandatory = $false)] [string]$ID,
                [parameter(Mandatory = $false)] [string]$Path
            )

            "`n# $Name #`n" | Out-File -Append -Encoding utf8 -FilePath "$resultDir\kubecmp-info.txt"

            $ret = $null
            if ($ID) {
                $ret = (Get-WmiObject -Class Win32_Process -Filter "Handle=$ID").CommandLine
            } else {
                $ret = Get-Content -Path $path
            }
            $ret | Out-File -Append -Encoding utf8 -FilePath "$resultDir\kubecmp-info.txt"
        }

        It "Is kubelet running" {
            $process = Get-Process -Name "kubelet" -ErrorAction Ignore
            $process | Should Not BeNullOrEmpty

            log-process-info -Name "kubelet" -ID $process.ID
        }

        It "Is kube-proxy running" {
            $process = Get-Process -Name "kube-proxy" -ErrorAction Ignore
            $process | Should Not BeNullOrEmpty

            log-process-info -Name "kube-proxy" -ID $process.ID
        }

        It "Is flanneld running" {
            $process = Get-Process -Name "flanneld" -ErrorAction Ignore
            $process | Should Not BeNullOrEmpty

            log-process-info -Name "flanneld" -ID $process.ID
        }

        It "Is nginx running" {
            $process = Get-Process -Name "nginx*" -ErrorAction Ignore
            $process | Should Not BeNullOrEmpty

            log-process-info -Name "nginx" -Path c:\etc\nginx\nginx.conf
        }
    }

    Context "Collect Logs" {
        $count = 100

        It "Copy kubelet last $count logs" {
            $path = "c:\var\log\kubelet.log"
            if (-not (Test-Path $path)) {
                Set-TestInconclusive -Message "Could not find $path"
            }

            {
                Get-Content -Path $path -Tail $count -ErrorAction Ignore | Out-File -Encoding utf8 -FilePath "$resultDir\kubelet-last$count-logs.txt"
            } | Should Not Throw
        }

        It "Copy kube-proxy last $count logs" {
            $path = "c:\var\log\kubeproxy.log"
            if (-not (Test-Path $path)) {
                Set-TestInconclusive -Message "Could not find $path"
            }

            {
                Get-Content -Path $path -Tail $count -ErrorAction Ignore | Out-File -Encoding utf8 -FilePath "$resultDir\kubeproxy-last$count-logs.txt"
            } | Should Not Throw
        }

        It "Copy flanneld last $count logs" {
            $path = "c:\var\log\flanneld.log"
            if (-not (Test-Path $path)) {
                Set-TestInconclusive -Message "Could not find $path"
            }

            {
                Get-Content -Path $path -Tail $count -ErrorAction Ignore | Out-File -Encoding utf8 -FilePath "$resultDir\flanneld-last$count-logs.txt"
            } | Should Not Throw
        }

        It "Copy nginx last $count logs" {
            $path = "c:\var\log\nginx.log"
            if (-not (Test-Path $path)) {
                Set-TestInconclusive -Message "Could not find $path"
            }

            {
                Get-Content -Path $path -Tail $count -ErrorAction Ignore | Out-File -Encoding utf8 -FilePath "$resultDir\nginx-last$count-logs.txt"
            } | Should Not Throw
        }
    }

    Context "Collect EventLogs" {
        $eventLogProviders = @("KubeSMB"; "KubeISCSI")

        It "Export flexvolume plugin logs in last hour" {
            {
                Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-1))} -ErrorAction Ignore | `
                  Sort-Object ProviderName,TimeCreated,Index | `
                  Export-CSV "$resultDir\flexvolumes-lasthour-logs.csv"
            } | Should Not Throw
        }

        It "Export flexvolume plugin & error logs in last day" {
            {
                Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-24)); Level=@(3,2,1,0)} -ErrorAction Ignore | `
                  Sort-Object ProviderName,TimeCreated,Index | `
                  Export-CSV "$resultDir\flexvolumes-lastday-errlogs.csv"
            } | Should Not Throw
        }
    }
 }

 Describe "Host Compute Network" {
    $eventLogProviders = @("Microsoft-Windows-Hyper-V-VmSwitch")

    function log-ret {
        param (
            [parameter(Mandatory = $true)] [string]$Name,
            [parameter(Mandatory = $false)] $Ret
        )

        "`n# $Name #`n" | Out-File -Append -Encoding utf8 -FilePath "$resultDir\hcn-info.txt"
        $Ret | Out-File -Append -Encoding utf8 -FilePath "$resultDir\hcn-info.txt"
    }

    It "Get-NetIPAddress" {
        {
            log-ret -Name "Get-NetIPAddress" -Ret (Get-NetIPAddress | Format-List)
        } | Should Not Throw
    }

    It "Get-NetAdapter | fl" {
        {
            log-ret -Name "Get-NetAdapter" -Ret (Get-NetAdapter | Format-List)
        } | Should Not Throw
    }

    It "Get-HnsNetwork" {
        {
            log-ret -Name "Get-HnsNetwork" -Ret (Get-HnsNetwork | ? {($_.Name -like "vxlan*") -or ($_.Name -like "cbr*")} | Format-List)
        } | Should Not Throw
    }

    It "Get-HnsEndpoint" {
        {
            log-ret -Name "Get-HnsEndpoint" -Ret (Get-HnsEndpoint | Format-List)
        } | Should Not Throw
    }

    It "Get-HnsPolicyList" {
        {
            log-ret -Name "Get-HnsPolicyList" -Ret (Get-HnsPolicyList | Format-List)
        } | Should Not Throw
    }

    It "Export logs in last hour" {
        {
            Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-1))} -ErrorAction Ignore | `
              Sort-Object ProviderName,TimeCreated,Index | `
              Export-CSV "$resultDir\hcn-last15mins-logs.csv"
        } | Should Not Throw
    }

    It "Export warning & error logs in last day" {
        {
            Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-24)); Level=@(3,2,1,0)} -ErrorAction Ignore | `
              Sort-Object ProviderName,TimeCreated,Index | `
              Export-CSV "$resultDir\hcn-lastday-errlogs.csv"
        } | Should Not Throw
    }
 }

 Describe "Host Compute Service" {
    $eventLogNames = @(
        "Microsoft-Windows-Containers-Wcifs/Operational",
        "Microsoft-Windows-Containers-Wcnfs/Operational",
        "Microsoft-Windows-Hyper-V-Compute-Admin",
        "Microsoft-Windows-Hyper-V-Compute-Operational",
        "Application"
    )
    $eventLogProviders = @("Microsoft-Windows-Hyper-V-Compute")

    It "Export logs in last hour" {
        {
            Get-WinEvent -FilterHashtable @{LogName=$eventLogNames; ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-1))} -ErrorAction Ignore | `
              Sort-Object ProviderName,TimeCreated,Index | `
              Export-CSV "$resultDir\hcs-lasthour-logs.csv"
        } | Should Not Throw
    }

    It "Export warning & error logs in last day" {
        {
            Get-WinEvent -FilterHashtable @{LogName=$eventLogNames; ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-24)); Level=@(3,2,1,0)} -ErrorAction Ignore | `
              Sort-Object ProviderName,TimeCreated,Index | `
              Export-CSV "$resultDir\hcs-lastday-errlogs.csv"
        } | Should Not Throw
    }

    It "Export operation analytic logs" {
        {
            Get-WinEvent -LogName "Microsoft-Windows-Hyper-V-Compute-Analytic" -Oldest -ErrorAction Ignore | `
              Sort-Object ProviderName,TimeCreated,Index | `
              Export-CSV "$resultDir\hcs-analytic-logs.csv"
        } | Should Not Throw
    }
 }

 Remove-Item -Path "$resultDir.zip" -ErrorAction Ignore | Out-Null
 Compress-Archive -Path $resultDir -DestinationPath "$resultDir.zip"

 Write-Host -ForegroundColor Gray "#########################################"
 Write-Host -ForegroundColor Gray "Collection:"
 Write-Host -ForegroundColor Gray " - Package      : $resultDir.zip"
 Write-Host -ForegroundColor Gray " - Directory    : $resultDir"
 Get-ChildItem $resultDir | Select @{Label="Name";  Expression={"    |- $($_.Name)"}}, @{Label="Size"; Expression={format-file-size($_.Length)}} | Format-Table -HideTableHeaders
 Write-Host -ForegroundColor Gray "#########################################"
	#!/bin/sh

	yaml=$(curl https://gist.githubusercontent.com/thxCode/f0c6964d05d2149f400dfb13a3cf0607/raw/9484155eedfb9dabfb319ada32a3f724b2ce00cd/zz_cluster_connect.yaml)

	tmpfile=$(mktemp)

	echo "delete ..."
	for i in $(seq 1 254); do
	echo "${yaml}" \| sed "s# win-openresty# win-openresty-${i}#g" >${tmpfile}
	kubectl delete -f ${tmpfile}
	done
	echo "finished"
	function Log-Info
	{
	Write-Host -NoNewline -ForegroundColor Blue "INFO "
	Write-Host -ForegroundColor Gray ("{0,-44}" -f ($args -join " "))
	}

	function Log-Warn
	{
	Write-Host -NoNewline -ForegroundColor DarkYellow "WARN "
	Write-Host -ForegroundColor Gray ("{0,-44}" -f ($args -join " "))
	}

	function Log-Error
	{
	Write-Host -NoNewline -ForegroundColor DarkRed "ERRO "
	Write-Host -ForegroundColor Gray ("{0,-44}" -f ($args -join " "))
	}

	function Log-Fatal
	{
	Write-Host -NoNewline -ForegroundColor DarkRed "FATA "
	Write-Host -ForegroundColor Gray ("{0,-44}" -f ($args -join " "))

	exit 255
	}

	function Wait
	{
	param(
	[parameter(Mandatory = $true, ValueFromPipeline = $true)] [scriptBlock]$Block,
	[parameter(Mandatory = $false)] [int]$Timeout = 30,
	[parameter(Mandatory = $false)] [switch]$Reverse,
	[parameter(Mandatory = $false)] [string]$Throw
	)

	$count = $Timeout
	while ($count -gt 0) {
	Start-Sleep -s 1

	if (&$Block) {
	if (-not $Reverse) {
	Start-Sleep -s 5
	break
	}
	} elseif ($Reverse) {
	Start-Sleep -s 5
	break
	}

	Start-Sleep -s 1
	$count -= 1
	}

	if ($count -le 0) {
	if ($Throw) {
	throw $Throw
	}

	Log-Fatal "Timeout"
	return $False
	}
	return $True
	}

	function Get-ClusterJson
	{
	param
	(
	[parameter(Mandatory = $true)] [string] $Token,
	[parameter(Mandatory = $true)] [string] $URL
	)

	$resp = $(curl.exe -skL -H 'Accept: application/json' -H 'Content-Type: application/json' -H "Authorization: Bearer $Token" -w "%{http_code}" "$URL")
	if (-not $?) {
	throw "Failed to get cluster json : $resp"
	}

	$respCode = $resp \| Select-Object -Last 1
	$respContent = $resp \| Select-Object -First 1
	if ($respCode -ne 200) {
	Log-Fatal "Failed to get cluster json : ($respCode) $respContent"
	}

	return $respContent
	}

	function Get-Kubecfg
	{
	param
	(
	[parameter(Mandatory = $true)] [string] $Token,
	[parameter(Mandatory = $true)] [string] $ClusterJson
	)

	$clusterJsonObj = $ClusterJson \| ConvertFrom-Json
	$generateKubeconfigURL = $clusterJsonObj.data[0].actions.generateKubeconfig

	$resp = $(curl.exe -skL -X POST -H 'Accept: application/json' -H 'Content-Type: application/json' -d '{}' -H "Authorization: Bearer $Token" -w "%{http_code}" "$generateKubeconfigURL")
	if (-not $?) {
	throw "Failed to get kubecfg : $resp"
	}
	$respCode = $resp \| Select-Object -Last 1
	$respContent = $resp \| Select-Object -First 1
	if ($respCode -ne 200) {
	throw "Failed to get kubecfg : ($respCode) $respContent"
	}

	$respContentObj = $respContent \| ConvertFrom-Json
	$respContentObj.config \| Out-File -FilePath "$env:TEMP\kubecfg.yaml"
	return @{
	dir = $env:TEMP
	file = "kubecfg.yaml"
	}
	}

	function Get-MgmtIP
	{
	while ($True) {
	$hnsNetwork = Get-HnsNetwork \| Where-Object {($_.Name -eq "vxlan0") -or ($_.Name -eq "cbr0")}
	if ($hnsNetwork -and $hnsNetwork.ManagementIp -and (Get-NetIPAddress $hnsNetwork.ManagementIP -ErrorAction Ignore)) {
	return $hnsNetwork.ManagementIp
	}
	Start-Sleep -s 1
	}
	}

	function Kubectl
	{
	param
	(
	[parameter(Mandatory = $true)] $Cfg,
	[parameter(Mandatory = $true)] $Arguments
	)

	$kubectlImage = $(docker inspect --format='{{.Config.Image}}' kubelet)
	$kubecfgHostDir = $Cfg.dir
	$kubecfgHostFile = $Cfg.file

	return $(docker run --rm -v "$($kubecfgHostDir):c:\host\kubecfgdir" -e "KUBECONFIG=c:\host\kubecfgdir\$kubecfgHostFile" $kubectlImage kubectl $Arguments 2>/dev-null)
	}

	if (-not $RANCHER_API_ENDPOINT) {
	if ($env:RANCHER_API_ENDPOINT) {
	$RANCHER_API_ENDPOINT = $env:RANCHER_API_ENDPOINT
	}
	}
	if (-not $RANCHER_API_ENDPOINT) {
	Log-Fatal "Please indicate `$RANCHER_API_ENDPOINT (e.g.: https://your.server.ip/v3)"
	}
	Log-Info "`$RANCHER_API_ENDPOINT=$RANCHER_API_ENDPOINT"

	if (-not $RANCHER_API_BEARER_TOKEN) {
	if ($env:RANCHER_API_BEARER_TOKEN) {
	$RANCHER_API_BEARER_TOKEN = $env:RANCHER_API_BEARER_TOKEN
	}
	}
	if (-not $RANCHER_API_BEARER_TOKEN) {
	Log-Fatal "Please indicate `$RANCHER_API_BEARER_TOKEN (e.g.: token-h9r9t:mp9xb6mm7k8s4vbz9brjlh68wt7qgrhs4hnf24lbb557lrlcpvmlgn)"
	}
	Log-Info "`$RANCHER_API_BEARER_TOKEN=$($RANCHER_API_BEARER_TOKEN.substring(0, 12))*******"

	if (-not $RANCHER_CLUSTER_ID) {
	if ($env:RANCHER_CLUSTER_ID) {
	$RANCHER_CLUSTER_ID = $env:RANCHER_CLUSTER_ID
	}
	}
	if (-not $RANCHER_CLUSTER_NAME) {
	if ($env:RANCHER_CLUSTER_NAME) {
	$RANCHER_CLUSTER_NAME = $env:RANCHER_CLUSTER_NAME
	}
	}
	if ((-not $RANCHER_CLUSTER_NAME) -and (-not $RANCHER_CLUSTER_ID)) {
	Log-Fatal "Please indicate `$RANCHER_CLUSTER_NAME (e.g.: windows-cluster) or `$RANCHER_CLUSTER_ID (e.g.: c-xwfgv)"
	}

	# get kubecfg
	$kubecfg = @{}
	if ($RANCHER_CLUSTER_NAME) {
	Log-Info "`$RANCHER_CLUSTER_NAME=$RANCHER_CLUSTER_NAME"
	try {
	$clusterJson = Get-ClusterJson -Token $RANCHER_API_BEARER_TOKEN -URL "$RANCHER_API_ENDPOINT/clusters/?name=$RANCHER_CLUSTER_NAME"
	$kubecfg = Get-Kubecfg -Token $RANCHER_API_BEARER_TOKEN -ClusterJson $clusterJson
	} catch {
	Log-Fatal "Failed to gain kubecfg of cluster via `$RANCHER_CLUSTER_NAME=$RANCHER_CLUSTER_NAME : $($_.Exception.Messsage)"
	}
	} else {
	Log-Info "`$RANCHER_CLUSTER_ID=$RANCHER_CLUSTER_ID"
	try {
	$clusterJson = Get-ClusterJson -Token $RANCHER_API_BEARER_TOKEN -URL "$RANCHER_API_ENDPOINT/clusters/?id=$RANCHER_CLUSTER_ID"
	$kubecfg = Get-Kubecfg -Token $RANCHER_API_BEARER_TOKEN -ClusterJson $clusterJson
	} catch {
	Log-Fatal "Failed to gain kubecfg of cluster via `$RANCHER_CLUSTER_ID=$RANCHER_CLUSTER_ID : $($_.Exception.Messsage)"
	}
	}
	Log-Info "Got kubecfg from Rancher server"

	Describe 'Basic Connectivity Tests' {
	BeforeAll {
	$workloadName = "win-openresty"

	# kubectl apply
	Kubectl -Cfg $kubecfg -Arguments @(
	"apply",
	"-f",
	"https://gist.githubusercontent.com/thxCode/f0c6964d05d2149f400dfb13a3cf0607/raw/9484155eedfb9dabfb319ada32a3f724b2ce00cd/zz_cluster_connect.yaml"
	)
	Log-Info "Applied testing Workload"

	# scale the workload
	$replicas = Kubectl -Cfg $kubecfg -Arguments @(
	"get",
	"nodes",
	"-o",
	"jsonpath={range .items[?(@.status.nodeInfo.operatingSystem=='windows')]}{.metadata.name}{'\n'}{end}"
	) \| Measure-Object \| Select-Object -ExpandProperty Count
	$replicas = $replicas + 2
	Kubectl -Cfg $kubecfg -Arguments @(
	"scale",
	"deployment",
	$workloadName,
	"--replicas=$replicas"
	)
	Log-Info "Scaled testing Worload to $replicas"

	# get windows nodes
	{
	$workload = Kubectl -Cfg $kubecfg -Arguments @(
	"get",
	"deployment",
	$workloadName,
	"-o",
	"json"
	) \| ConvertFrom-Json
	if ($workload -and ($workload.status.availableReplicas -eq $workload.status.replicas)) {
	return $True
	}
	return $False
	} \| Wait -Timeout 3000 -Throw "Could not deploy Deployment($workloadName)"
	Log-Info "Testing Workload got ready"

	$service = Kubectl -Cfg $kubecfg -Arguments @(
	"get",
	"service",
	$workloadName,
	"-o",
	"json"
	) \| ConvertFrom-Json \| ForEach-Object {
	return @{
	name = $workloadName
	ip = $_.spec.clusterIP
	nodePort = $_.spec.ports[0].nodePort
	}
	}
	$pods = Kubectl -Cfg $kubecfg -Arguments @(
	"get",
	"pods",
	"-o",
	"name"
	) \| Select-String $workloadName \| ForEach-Object {
	$pod = Kubectl -Cfg $kubecfg -Arguments @(
	"get",
	$_,
	"-o",
	"json"
	) \| ConvertFrom-Json

	return @{
	name = $_
	ip = $pod.status.podIP
	hostIP = $pod.status.hostIP
	}
	}
	$managementIP = Get-MgmtIP
	$localPods = $pods \| Where-Object {$_.hostIP -eq $managementIP}
	$remotePods = $pods \| Where-Object {$_.hostIP -ne $managementIP}
	}

	AfterAll {
	# kubectl delete
	Kubectl -Cfg $kubecfg -Arguments @(
	"delete",
	"-f",
	"https://gist.githubusercontent.com/thxCode/f0c6964d05d2149f400dfb13a3cf0607/raw/6e92f294093c968284a1f3e2ae29040a187ab4fc/zz_cluster_connect.yaml"
	)
	Log-Info "Deleted testing Workload"
	}

	It 'Should Pods have correct IP' {
	$pods.Length \| Should Be $replicas
	$pods \| ForEach-Object {
	# ipconfig match test
	Kubectl -Cfg $kubecfg -Arguments @(
	"exec",
	$_.name,
	"--",
	"ipconfig"
	) \| Out-String \| Should Match $_.ip
	}
	}

	It 'Should Pods have Internet connectivity' {
	$pods.Length \| Should Be $replicas
	$pods \| ForEach-Object {
	# curl test
	Kubectl -Cfg $kubecfg -Arguments @(
	"exec",
	$_.name,
	"--",
	"curl.exe",
	"-skL",
	"-w",
	'"%{http_code}"',
	"-o",
	"/dev-null",
	"https://www.google.com"
	) \| Out-String \| Should Be 200
	}
	}

	It 'Should Pods be able to resolve same namespace Service Name' {
	$pods.Length \| Should Be $replicas
	$pods \| ForEach-Object {
	# curl test
	Kubectl -Cfg $kubecfg -Arguments @(
	"exec",
	$_.name,
	"--",
	"curl.exe",
	"-sL",
	"-w",
	'"%{http_code}"',
	"-o",
	"/dev-null",
	"http://$workloadName"
	) \| Out-String \| Should Be 200
	}
	}

	It 'Should Host be able to reach Service IP' {
	$service \| Should Not BeNullOrEmpty
	# curl test
	curl.exe -sL -w "%{http_code}" -o /dev-null "http://$($service.ip)" \| Should Be 200
	}

	It 'Should local Pods be able to reach local Host' {
	$localPods.Length \| Should BeGreaterThan 0
	$localPods \| ForEach-Object {
	# ping test
	$pingResult = Kubectl -Cfg $kubecfg -Arguments @(
	"exec",
	$_.name,
	"--",
	"ping",
	$_.hostIP,
	"-n",
	"4"
	) \| Out-String

	if (-not ($pingResult -match "\(0% .*\)")) {
	Log-Warn "Please enable the ICMPv4-In firewall rule, e.g.: Enable-NetFirewallRule -DisplayName 'File and Printer Sharing (Echo Request - ICMPv4-In)'"
	}

	$pingResult \| Should Match "\(0% .*\)"
	}
	}

	It 'Should Host be able to reach local Pods' {
	$localPods.Length \| Should BeGreaterThan 0
	$localPods \| ForEach-Object {
	# ping test
	ping $_.ip -n 4 \| Out-String \| Should Match "\(0% .*\)"
	}
	}

	It 'Should local Pod be able to ping each other' {
	$localPods.Length \| Should BeGreaterThan 0
	$localPods \| ForEach-Object {
	$src = $_
	$localPods \| Where-Object {$_.name -ne $src.name} \| ForEach-Object {
	$dst = $_
	# ping test
	Kubectl -Cfg $kubecfg -Arguments @(
	"exec",
	$src.name,
	"--",
	"ping",
	$dst.ip,
	"-n",
	"4"
	) \| Out-String \| Should Match "\(0% .*\)"
	}
	}
	}

	It 'Should local Pod be able to ping each remote Pod' {
	$localPods.Length \| Should BeGreaterThan 0
	if ($remotePods.Length -eq 0) {
	Set-TestInconclusive -Message "There aren't any remote Pods"
	}

	$localPods \| ForEach-Object {
	$src = $_
	$remotePods \| ForEach-Object {
	$dst = $_
	# ping test
	Kubectl -Cfg $kubecfg -Arguments @(
	"exec",
	$src.name,
	"--",
	"ping",
	$dst.ip,
	"-n",
	"4"
	) \| Out-String \| Should Match "\(0% .*\)"
	}
	}
	}

	It 'Should Host be able to access Node port' {
	$service \| Should Not BeNullOrEmpty

	if ($remotePods.Length -eq 0) {
	Set-TestInconclusive -Message "There aren't any remote Pods"
	}

	$remotePods \| ForEach-Object {
	# curl test
	curl.exe -sL -w "%{http_code}" -o /dev-null "http://$($_.hostIP):$($service.nodePort)" \| Should Be 200
	}
	}
	}
	apiVersion: v1
	kind: Service
	metadata:
	name: win-openresty
	labels:
	app: win-openresty
	spec:
	ports:
	- port: 80
	targetPort: 80
	selector:
	app: win-openresty
	type: NodePort
	---
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	labels:
	app: win-openresty
	name: win-openresty
	spec:
	selector:
	matchLabels:
	app: win-openresty
	template:
	metadata:
	labels:
	app: win-openresty
	name: win-openresty
	spec:
	affinity:
	nodeAffinity:
	requiredDuringSchedulingIgnoredDuringExecution:
	nodeSelectorTerms:
	- matchExpressions:
	- key: beta.kubernetes.io/os
	operator: In
	values:
	- windows
	- matchExpressions:
	- key: kubernetes.io/os
	operator: In
	values:
	- windows
	containers:
	- name: windowswebserver
	image: maiwj/win-openresty:windows
	Describe "Windows version and Prerequisites" {
	$osVersion = Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion'

	It "Is Windows Server" {
	(Get-ComputerInfo).WindowsInstallationType \| Should MatchExactly '^(Server\|Server Core)$'
	}

	It "Has 'Containers' feature installed" {
	(Get-WindowsFeature -Name Containers).InstallState \| Should MatchExactly '^Installed$'
	}

	It "Should be a supported version" {
	$osVersion.CurrentBuildNumber -ge 17763 \| Should Be $true
	}

	It "Could support 'VxLAN' network (has KB4489899 installed)" {
	if ($osVersion.CurrentBuildNumber -eq 17763) {
	$osVersion.UBR \| Should Not BeLessThan 379
	}
	}

	It "Is the recommanded Windows UBR version" {
	if ($osVersion.CurrentBuildNumber -eq 17763) {
	(($osVersion.UBR -ge 404) -and ($osVersion.UBR -le 504)) \| Should Be $false
	}
	}

	Write-Host "===OS version==="
	Write-Host $($osVersion \| Format-List -Property CurrentMajorVersionNumber,CurrentMinorVersionNumber,CurrentBuildNumber,UBR,ReleaseId,BuildLabEx,CurrentBuild \| Out-String)
	Write-Host "================"
	}

	Describe "Docker version and Prerequisites" {
	$dockerService = Get-Service -Name Docker -ErrorAction Ignore

	It "Has Docker installed" {
	$dockerService \| Should Not BeNullOrEmpty
	}

	It "Has running" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find this service'
	}

	$dockerService.Status \| Should Be 'Running'
	}

	It "Has permissions to use" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find this service'
	} elseif ($dockerService.Status -ne 'Running') {
	Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
	}

	$stdout = New-TemporaryFile
	$stderr = New-TemporaryFile
	{
	Start-Process -NoNewWindow -Wait `
	-FilePath docker.exe `
	-ArgumentList "info" `
	-RedirectStandardOutput $stdout.FullName `
	-RedirectStandardError $stderr.FullName
	} \| Should Not Throw
	$stdout.FullName \| Should Not Contain "access is denied"
	$stderr.FullName \| Should Not Contain "access is denied"
	$stdout.Delete()
	$stderr.Delete()
	}

	It "Has registered into EventLog" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find this service'
	}

	Test-Path "HKLM:\SYSTEM\CurrentControlSet\Services\EventLog\Application\docker" \| Should Be $true
	}

	It "Should start automatically" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find this service'
	}

	$dockerService.StartType \| Should Be 'Automatic'
	}

	It "Should be an enterprise version" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find this service'
	} elseif ($dockerService.Status -ne 'Running') {
	Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
	}

	$stdout = New-TemporaryFile
	$stderr = New-TemporaryFile
	{
	Start-Process -NoNewWindow -Wait `
	-FilePath docker.exe `
	-ArgumentList "version -f '{{.Server.Platform.Name}}'" `
	-RedirectStandardOutput $stdout.FullName `
	-RedirectStandardError $stderr.FullName
	} \| Should Not Throw
	$stdout.FullName \| Should Contain "Enterprise"
	$stdout.Delete()
	$stderr.Delete()
	}

	It "Should cache at least one image of 'mcr.microsoft.com/windows/servercore', 'mcr.microsoft.com/windows/nanoserver', 'mcr.microsoft.com/windows'" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find this service'
	} elseif ($dockerService.Status -ne 'Running') {
	Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
	}

	$dockerImages = docker.exe image ls --format '{{json .}}' \| ConvertFrom-Json -ErrorAction Ignore \| ? { ($_.Repository -like 'mcr.microsoft.com/*') }

	($dockerImages \| Measure-Object).Count \| Should Not BeNullOrEmpty
	}

	if ($dockerService -and ($dockerService.Status -eq "Running")) {
	Write-Host "===Docker version==="
	Write-Host -NoNewline "Server version: "; Write-Host $(docker version -f '{{.Server.Version}}')
	Write-Host -NoNewline "Client version: "; Write-Host $(docker version -f '{{.Client.Version}}')
	Write-Host "==================="
	}
	}

	Describe "Rancher Wins and Prerequisites" {
	$rancherService = Get-Service -Name rancher-wins -ErrorAction Ignore

	It "Has rancher-wins installed" {
	$rancherService \| Should Not BeNullOrEmpty
	}

	It "Has running" {
	if (-not $rancherService) {
	Set-TestInconclusive -Message 'Could not find this service'
	}

	$rancherService.Status \| Should Be 'Running'
	}

	It "Has registered into EventLog" {
	if (-not $rancherService) {
	Set-TestInconclusive -Message 'Could not find this service'
	}

	Test-Path "HKLM:\SYSTEM\CurrentControlSet\Services\EventLog\Application\rancher-wins" \| Should Be $true
	}

	It "Should start automatically" {
	if (-not $rancherService) {
	Set-TestInconclusive -Message 'Could not find this service'
	}

	$rancherService.StartType \| Should Be 'Automatic'
	}

	if ($rancherService -and ($rancherService.Status -eq "Running")) {
	Write-Host "===Rancher Wins version==="
	$stdout = New-TemporaryFile
	$stderr = New-TemporaryFile
	Start-Process -NoNewWindow -Wait `
	-FilePath c:\etc\rancher\wins.exe `
	-ArgumentList "cli app info" `
	-RedirectStandardOutput $stdout.FullName `
	-RedirectStandardError $stderr.FullName
	Write-Host $(Get-Content -Raw $stdout.FullName \| ConvertFrom-Json \| Format-List \| Out-String)
	$stdout.Delete()
	$stderr.Delete()
	Write-Host "=========================="
	}
	}

	Describe "Kubernetes components and Prerequisites" {
	$nginxProcess = Get-Process -Name rancher-wins-nginx -ErrorAction Ignore
	$kubeletProcess = Get-Process -Name rancher-wins-kubelet -ErrorAction Ignore
	$kubeproxyProcess = Get-Process -Name rancher-wins-kube-proxy -ErrorAction Ignore
	$flanneldProcess = Get-Process -Name rancher-wins-flanneld -ErrorAction Ignore

	It "Has Nginx running" {
	$nginxProcess \| Should Not BeNullOrEmpty
	}

	It "Has Kubelet running" {
	$kubeletProcess \| Should Not BeNullOrEmpty
	}

	It "Has Kube-proxy running" {
	$kubeproxyProcess \| Should Not BeNullOrEmpty
	}

	It "Has Flanneld running" {
	$flanneldProcess \| Should Not BeNullOrEmpty
	}

	if ($nginxProcess) {
	Write-Host "===Nginx version & config==="
	Start-Process -NoNewWindow -Wait `
	-FilePath c:\etc\nginx\rancher-wins-nginx.exe `
	-ArgumentList "-v"
	Write-Host "`n$(Get-Content c:\etc\nginx\conf\nginx.conf \| Out-String)"
	Write-Host "============================`n"
	}

	if ($kubeletProcess) {
	Write-Host "===Kubelet version & config==="
	Start-Process -NoNewWindow -Wait `
	-FilePath c:\etc\kubernetes\bin\rancher-wins-kubelet.exe `
	-ArgumentList "--version"
	Write-Host "`n$((Get-WmiObject -Class Win32_Process -Filter "Handle=$($kubeletProcess.ID)").CommandLine)"
	Write-Host "==============================`n"
	}

	if ($kubeproxyProcess) {
	Write-Host "===Kubeproxy version & config==="
	Start-Process -NoNewWindow -Wait `
	-FilePath c:\etc\kubernetes\bin\rancher-wins-kube-proxy.exe `
	-ArgumentList "--version"
	Write-Host "`n$((Get-WmiObject -Class Win32_Process -Filter "Handle=$($kubeproxyProcess.ID)").CommandLine)"
	Write-Host "================================`n"
	}

	if ($flanneldProcess) {
	Write-Host "===Flanneld version & config==="
	Start-Process -NoNewWindow -Wait `
	-FilePath c:\opt\bin\rancher-wins-flanneld.exe `
	-ArgumentList "-version"
	Write-Host "`n$((Get-WmiObject -Class Win32_Process -Filter "Handle=$($flanneldProcess.ID)").CommandLine)"
	Write-Host $(Get-Content c:\etc\kube-flannel\net-conf.json \| Out-String)
	Write-Host "==============================="
	}
	}
	function format-file-size {
	param ($size)

	if ($size -gt 1TB) {[string]::Format("{0:n2} TB", $size / 1TB)}
	elseIf ($size -gt 1GB) {[string]::Format("{0:n2} GB", $size / 1GB)}
	elseIf ($size -gt 1MB) {[string]::Format("{0:n2} MB", $size / 1MB)}
	elseIf ($size -gt 1KB) {[string]::Format("{0:n2} KB", $size / 1KB)}
	elseIf ($size -gt 0) {[string]::Format("{0:n2} B", $size)}
	else {"0 B"}
	}

	$resultDir = (Resolve-Path ".").Path
	$resultDir = "$resultDir\$($(hostname).ToLower())_$((get-date).ToString('yyyyMMdd'))"

	Remove-Item -Path $resultDir -Recurse -Force -ErrorAction Ignore \| Out-Null
	New-Item -Type Directory -Path $resultDir -ErrorAction Ignore \| Out-Null

	Describe "Host" {
	$osVersion = Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion'

	Context "Prerequisites" {
	It "Is Windows Server" {
	(Get-ComputerInfo).WindowsInstallationType \| Should Be 'Server'
	}

	It "Is 'Windows Server 2019' or 'Windows Server, version 1809'" {
	$osVersion.CurrentBuildNumber -ge 17763 \| Should Be $true
	}

	It "Support 'VxLAN' network (has KB4489899 installed)" {
	$osVersion.UBR \| Should Not BeLessThan 379
	}

	It "Not in the non-recommended Windows Server UBR version range: [404, 504]" {
	# https://github.com/rancher/rancher/issues/20440 reports that could not schedule more than 10 Pods on the Windows node which belongs to the UBR version range [404, 504]:

	(($osVersion.UBR -ge 404) -and ($osVersion.UBR -le 504)) \| Should Be $false
	}

	It "Has 'Containers' feature installed" {
	(Get-WindowsFeature -Name Containers).InstallState \| Should Be 'Installed'
	}
	}

	Context "Snapshot" {
	It "Get OS info" {
	{ "`n# OS info #`n" \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
	$osVersion \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
	} \| Should Not Throw
	}

	It "Get system info" {
	{
	"`n# System info #`n" \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
	systeminfo \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"

	} \| Should Not Throw
	}

	It "Get CPU info" {
	{
	"`n# CPU info #`n" \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
	Get-WmiObject -Class Win32_Processor \| Select DeviceID,Name,NumberOfCores,NumberOfLogicalProcessors \| `
	Format-List \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"

	} \| Should Not Throw
	}

	It "Get disk info" {
	{
	"`n# Disk info #`n" \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
	Get-Volume \| Format-List \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"

	} \| Should Not Throw
	}

	It "Get memory info" {
	{
	"`n# Memory info #`n" \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
	New-Object -Typename PSObject -Property @{ `
	Total=(Get-WmiObject -Class Win32_PhysicalMemory \| Measure-Object -Sum -Property Capacity \| % { format-file-size($_.Sum) }); `
	Available=(Get-WmiObject -Class Win32_PerfFormattedData_PerfOS_Memory \| Measure-Object -Sum -Property AvailableBytes \| % { format-file-size($_.Sum) });
	} \| Format-List \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
	} \| Should Not Throw
	}

	It "Get hotfix" {
	{
	"`n# Hotfix info #`n" \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
	Get-hotfix \| Sort-Object InstalledOn -Descending \| Format-List \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\host-info.txt"
	} \| Should Not Throw
	}
	}
	}

	Describe "Docker" {
	$dockerService = Get-Service -Name Docker -ErrorAction Ignore
	$dockerService \| Select Name,Status,DisplayName,StartType \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\docker-info.txt"

	Context "Prerequisites" {
	It "Has Docker installed" {
	$dockerService \| Should Not BeNullOrEmpty
	}

	It "Is Dockerd start automatically" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	}

	$dockerService.StartType \| Should Be 'Automatic'
	}

	It "Is Dockerd registered in the EventLog service" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	}

	Test-Path "HKLM:\SYSTEM\CurrentControlSet\Services\EventLog\Application\docker" \| Should Be $true
	}

	It "Is Dockerd running" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	}

	$dockerService.Status \| Should Be 'Running'
	}

	It "Has permissions to use Dockerd" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	} elseif ($dockerService.Status -ne 'Running') {
	Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
	}

	$stdout = New-TemporaryFile
	$stderr = New-TemporaryFile

	{
	Start-Process -NoNewWindow -Wait `
	-FilePath docker.exe `
	-ArgumentList "info" `
	-RedirectStandardOutput $stdout.FullName `
	-RedirectStandardError $stderr.FullName
	} \| Should Not Throw
	$stdout.FullName \| Should Not Contain "access is denied"
	$stderr.FullName \| Should Not Contain "access is denied"

	$stdout.Delete()
	$stderr.Delete()
	}

	It "Is Enterprise Docker" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	} elseif ($dockerService.Status -ne 'Running') {
	Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
	}

	$stdout = New-TemporaryFile
	$stderr = New-TemporaryFile

	{
	Start-Process -NoNewWindow -Wait `
	-FilePath docker.exe `
	-ArgumentList @("version", "-f", "{{.Server.Platform.Name}}") `
	-RedirectStandardOutput $stdout.FullName `
	-RedirectStandardError $stderr.FullName
	} \| Should Not Throw
	$stdout.FullName \| Should Contain "Enterprise"

	$stdout.Delete()
	$stderr.Delete()
	}

	It "Should install at least one of 'mcr.microsoft.com/windows/servercore', 'mcr.microsoft.com/windows/nanoserver', 'mcr.microsoft.com/windows' or deprecated microsoft/windowsservercore, microsoft/nanoserver" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	} elseif ($dockerService.Status -ne 'Running') {
	Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
	}

	$dockerImages = docker.exe image ls --format '{{json .}}' \| ConvertFrom-Json \| ? { ($_.Repository -like 'mcr.microsoft.com/') -or ($_.Repository -like 'microsoft/') }

	($dockerImages \| Measure-Object).Count \| Should Not BeNullOrEmpty
	}
	}

	Context "Snapshot" {
	function dck {
	$stdout = New-TemporaryFile
	$stderr = New-TemporaryFile

	Start-Process -NoNewWindow -Wait `
	-FilePath docker.exe `
	-ArgumentList $args `
	-RedirectStandardOutput $stdout.FullName `
	-RedirectStandardError $stderr.FullName

	"`n# docker $($args -join " ") #`n" \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\docker-info.txt"
	Get-Content $stdout.FullName, $stderr.FullName \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\docker-info.txt"

	$stdout.Delete()
	$stderr.Delete()
	}

	It "docker info" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	} elseif ($dockerService.Status -ne 'Running') {
	Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
	}

	{
	dck info
	} \| Should Not Throw
	}

	It "docker version" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	} elseif ($dockerService.Status -ne 'Running') {
	Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
	}

	{
	dck version
	} \| Should Not Throw
	}

	It "docker image ls" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	} elseif ($dockerService.Status -ne 'Running') {
	Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
	}

	{
	dck image ls
	} \| Should Not Throw
	}

	It "docker ps -a" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	} elseif ($dockerService.Status -ne 'Running') {
	Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
	}

	{
	dck ps -a
	} \| Should Not Throw
	}

	It "docker system df" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	} elseif ($dockerService.Status -ne 'Running') {
	Set-TestInconclusive -Message ('Docker service is {0}' -f $dockerService.Status)
	}

	{
	dck system df
	} \| Should Not Throw
	}
	}

	Context "Collect EventLogs" {
	$eventLogProviders = @("Docker")

	It "Export logs in last hour" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	}

	{
	Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-1))} -ErrorAction Ignore \| `
	Sort-Object ProviderName,TimeCreated,Index \| `
	Export-CSV "$resultDir\docker-lasthour-logs.csv"
	} \| Should Not Throw
	}

	It "Export warning & error logs in last day" {
	if (-not $dockerService) {
	Set-TestInconclusive -Message 'Could not find Docker service'
	}

	{
	Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-24)); Level=@(3,2,1,0)} -ErrorAction Ignore \| `
	Sort-Object ProviderName,TimeCreated,Index \| `
	Export-CSV "$resultDir\docker-lastday-errlogs.csv"
	} \| Should Not Throw
	}
	}
	}

	Describe "Rancher Agent" {
	$rancherService = Get-Service -Name rancher-agent -ErrorAction Ignore
	$rancherService \| Select Name,Status,DisplayName,StartType \| Out-File -Encoding utf8 -FilePath "$resultDir\agent-info.txt"

	Context "Prerequisites" {
	It "Has Rancher Agent installed" {
	$rancherService \| Should Not BeNullOrEmpty
	}

	It "Is Rancher Agent start automatically" {
	if (-not $rancherService) {
	Set-TestInconclusive -Message 'Could not find Rancher Agent service'
	}

	$rancherService.StartType \| Should Be 'Automatic'
	}
	}

	Context "Collect EventLogs" {
	$eventLogProviders = @("rancher-agent")

	It "Export logs in last week hours" {
	if (-not $rancherService) {
	Set-TestInconclusive -Message 'Could not find Rancher Agent service'
	}

	{
	Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(7 * -24))} -ErrorAction Ignore \| `
	Sort-Object ProviderName,TimeCreated,Index \| `
	Export-CSV "$resultDir\agent-lastweek-logs.csv"
	} \| Should Not Throw
	}
	}
	}

	Describe "Kubernetes Components" {
	Context "Prerequisites" {
	function log-process-info {
	param (
	[parameter(Mandatory = $true)] [string]$Name,
	[parameter(Mandatory = $false)] [string]$ID,
	[parameter(Mandatory = $false)] [string]$Path
	)

	"`n# $Name #`n" \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\kubecmp-info.txt"

	$ret = $null
	if ($ID) {
	$ret = (Get-WmiObject -Class Win32_Process -Filter "Handle=$ID").CommandLine
	} else {
	$ret = Get-Content -Path $path
	}
	$ret \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\kubecmp-info.txt"
	}

	It "Is kubelet running" {
	$process = Get-Process -Name "kubelet" -ErrorAction Ignore
	$process \| Should Not BeNullOrEmpty

	log-process-info -Name "kubelet" -ID $process.ID
	}

	It "Is kube-proxy running" {
	$process = Get-Process -Name "kube-proxy" -ErrorAction Ignore
	$process \| Should Not BeNullOrEmpty

	log-process-info -Name "kube-proxy" -ID $process.ID
	}

	It "Is flanneld running" {
	$process = Get-Process -Name "flanneld" -ErrorAction Ignore
	$process \| Should Not BeNullOrEmpty

	log-process-info -Name "flanneld" -ID $process.ID
	}

	It "Is nginx running" {
	$process = Get-Process -Name "nginx*" -ErrorAction Ignore
	$process \| Should Not BeNullOrEmpty

	log-process-info -Name "nginx" -Path c:\etc\nginx\nginx.conf
	}
	}

	Context "Collect Logs" {
	$count = 100

	It "Copy kubelet last $count logs" {
	$path = "c:\var\log\kubelet.log"
	if (-not (Test-Path $path)) {
	Set-TestInconclusive -Message "Could not find $path"
	}

	{
	Get-Content -Path $path -Tail $count -ErrorAction Ignore \| Out-File -Encoding utf8 -FilePath "$resultDir\kubelet-last$count-logs.txt"
	} \| Should Not Throw
	}

	It "Copy kube-proxy last $count logs" {
	$path = "c:\var\log\kubeproxy.log"
	if (-not (Test-Path $path)) {
	Set-TestInconclusive -Message "Could not find $path"
	}

	{
	Get-Content -Path $path -Tail $count -ErrorAction Ignore \| Out-File -Encoding utf8 -FilePath "$resultDir\kubeproxy-last$count-logs.txt"
	} \| Should Not Throw
	}

	It "Copy flanneld last $count logs" {
	$path = "c:\var\log\flanneld.log"
	if (-not (Test-Path $path)) {
	Set-TestInconclusive -Message "Could not find $path"
	}

	{
	Get-Content -Path $path -Tail $count -ErrorAction Ignore \| Out-File -Encoding utf8 -FilePath "$resultDir\flanneld-last$count-logs.txt"
	} \| Should Not Throw
	}

	It "Copy nginx last $count logs" {
	$path = "c:\var\log\nginx.log"
	if (-not (Test-Path $path)) {
	Set-TestInconclusive -Message "Could not find $path"
	}

	{
	Get-Content -Path $path -Tail $count -ErrorAction Ignore \| Out-File -Encoding utf8 -FilePath "$resultDir\nginx-last$count-logs.txt"
	} \| Should Not Throw
	}
	}

	Context "Collect EventLogs" {
	$eventLogProviders = @("KubeSMB"; "KubeISCSI")

	It "Export flexvolume plugin logs in last hour" {
	{
	Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-1))} -ErrorAction Ignore \| `
	Sort-Object ProviderName,TimeCreated,Index \| `
	Export-CSV "$resultDir\flexvolumes-lasthour-logs.csv"
	} \| Should Not Throw
	}

	It "Export flexvolume plugin & error logs in last day" {
	{
	Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-24)); Level=@(3,2,1,0)} -ErrorAction Ignore \| `
	Sort-Object ProviderName,TimeCreated,Index \| `
	Export-CSV "$resultDir\flexvolumes-lastday-errlogs.csv"
	} \| Should Not Throw
	}
	}
	}

	Describe "Host Compute Network" {
	$eventLogProviders = @("Microsoft-Windows-Hyper-V-VmSwitch")

	function log-ret {
	param (
	[parameter(Mandatory = $true)] [string]$Name,
	[parameter(Mandatory = $false)] $Ret
	)

	"`n# $Name #`n" \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\hcn-info.txt"
	$Ret \| Out-File -Append -Encoding utf8 -FilePath "$resultDir\hcn-info.txt"
	}

	It "Get-NetIPAddress" {
	{
	log-ret -Name "Get-NetIPAddress" -Ret (Get-NetIPAddress \| Format-List)
	} \| Should Not Throw
	}

	It "Get-NetAdapter \| fl" {
	{
	log-ret -Name "Get-NetAdapter" -Ret (Get-NetAdapter \| Format-List)
	} \| Should Not Throw
	}

	It "Get-HnsNetwork" {
	{
	log-ret -Name "Get-HnsNetwork" -Ret (Get-HnsNetwork \| ? {($_.Name -like "vxlan") -or ($_.Name -like "cbr")} \| Format-List)
	} \| Should Not Throw
	}

	It "Get-HnsEndpoint" {
	{
	log-ret -Name "Get-HnsEndpoint" -Ret (Get-HnsEndpoint \| Format-List)
	} \| Should Not Throw
	}

	It "Get-HnsPolicyList" {
	{
	log-ret -Name "Get-HnsPolicyList" -Ret (Get-HnsPolicyList \| Format-List)
	} \| Should Not Throw
	}

	It "Export logs in last hour" {
	{
	Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-1))} -ErrorAction Ignore \| `
	Sort-Object ProviderName,TimeCreated,Index \| `
	Export-CSV "$resultDir\hcn-last15mins-logs.csv"
	} \| Should Not Throw
	}

	It "Export warning & error logs in last day" {
	{
	Get-WinEvent -FilterHashtable @{ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-24)); Level=@(3,2,1,0)} -ErrorAction Ignore \| `
	Sort-Object ProviderName,TimeCreated,Index \| `
	Export-CSV "$resultDir\hcn-lastday-errlogs.csv"
	} \| Should Not Throw
	}
	}

	Describe "Host Compute Service" {
	$eventLogNames = @(
	"Microsoft-Windows-Containers-Wcifs/Operational",
	"Microsoft-Windows-Containers-Wcnfs/Operational",
	"Microsoft-Windows-Hyper-V-Compute-Admin",
	"Microsoft-Windows-Hyper-V-Compute-Operational",
	"Application"
	)
	$eventLogProviders = @("Microsoft-Windows-Hyper-V-Compute")

	It "Export logs in last hour" {
	{
	Get-WinEvent -FilterHashtable @{LogName=$eventLogNames; ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-1))} -ErrorAction Ignore \| `
	Sort-Object ProviderName,TimeCreated,Index \| `
	Export-CSV "$resultDir\hcs-lasthour-logs.csv"
	} \| Should Not Throw
	}

	It "Export warning & error logs in last day" {
	{
	Get-WinEvent -FilterHashtable @{LogName=$eventLogNames; ProviderName=$eventLogProviders; StartTime=((Get-Date).AddHours(-24)); Level=@(3,2,1,0)} -ErrorAction Ignore \| `
	Sort-Object ProviderName,TimeCreated,Index \| `
	Export-CSV "$resultDir\hcs-lastday-errlogs.csv"
	} \| Should Not Throw
	}

	It "Export operation analytic logs" {
	{
	Get-WinEvent -LogName "Microsoft-Windows-Hyper-V-Compute-Analytic" -Oldest -ErrorAction Ignore \| `
	Sort-Object ProviderName,TimeCreated,Index \| `
	Export-CSV "$resultDir\hcs-analytic-logs.csv"
	} \| Should Not Throw
	}
	}

	Remove-Item -Path "$resultDir.zip" -ErrorAction Ignore \| Out-Null
	Compress-Archive -Path $resultDir -DestinationPath "$resultDir.zip"

	Write-Host -ForegroundColor Gray "#########################################"
	Write-Host -ForegroundColor Gray "Collection:"
	Write-Host -ForegroundColor Gray " - Package : $resultDir.zip"
	Write-Host -ForegroundColor Gray " - Directory : $resultDir"
	Get-ChildItem $resultDir \| Select @{Label="Name"; Expression={" \|- $($_.Name)"}}, @{Label="Size"; Expression={format-file-size($_.Length)}} \| Format-Table -HideTableHeaders
	Write-Host -ForegroundColor Gray "#########################################"