Last active
August 3, 2023 20:59
-
-
Save alexeldeib/37ca35e15b9022847aefd8dbe16cc2dd to your computer and use it in GitHub Desktop.
AKS Nvidia cgroupv2 repro
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| export GROUP=ace-mig | |
| export NAME=ace-mig | |
| export LOCATION=eastus | |
| az group create -g "${GROUP}" -l ${LOCATION} | |
| # create a cluster with a default pool with some typical parameters. | |
| # not really relevant. | |
| # only key piece is use k8s version >= 1.25.0 for Ubuntu 22.04 with cgroupv2. | |
| # can check available versions with | |
| # az aks get-versions -l eastus -o table | |
| az aks create --generate-ssh-keys -g $GROUP -n "${NAME}" -l $LOCATION --node-vm-size Standard_D4ads_v5 -c 2 -k 1.26.3 --vm-set-type VirtualMachineScaleSets --load-balancer-sku Standard --node-osdisk-type Managed --node-osdisk-size 512 --enable-managed-identity --network-plugin kubenet | |
| # grab the kubeconfig | |
| az aks get-credentials -g $GROUP -n $NAME | |
| # create a gpu pool. I used this size, any would apply. | |
| az aks nodepool add -g "$GROUP" --cluster-name "$NAME" -n nca100 --node-osdisk-type Managed --node-osdisk-size 512 --node-vm-size Standard_NC24ads_A100_v4 -c 1 | |
| # install the nvidia device plugin | |
| kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/main/deployments/static/nvidia-device-plugin-compat-with-cpumanager.yml | |
| # my nodes, as reference | |
| # kubectl get node | |
| # NAME STATUS ROLES AGE VERSION | |
| # aks-nca100-36400834-vmss000000 Ready agent 58m v1.26.3 | |
| # aks-nodepool1-32772828-vmss000000 Ready agent 64m v1.26.3 | |
| # aks-nodepool1-32772828-vmss000001 Ready agent 64m v1.26.3 | |
| # ssh OR nsenter node using node-shell + privileged pod | |
| # tried both to eliminate any container mount issues. | |
| # same behavior | |
| # https://github.com/kvaps/kubectl-node-shell | |
| kubectl node-shell aks-nca100-36400834-vmss000000 | |
| # now in node as root, via node-shell or separately via SSH | |
| nvidia-ctk system create-dev-char-symlinks --create-all | |
| # in a separate shell: | |
| # write out the test manifest | |
| tee nvidia-smi-loop.yaml > /dev/null <<'EOF' | |
| apiVersion: v1 | |
| kind: Pod | |
| metadata: | |
| name: cuda-nvidia-smi-loop | |
| spec: | |
| restartPolicy: OnFailure | |
| containers: | |
| - name: cuda | |
| image: "nvcr.io/nvidia/cuda:12.0.0-base-ubuntu20.04" | |
| command: ["/bin/sh", "-c"] | |
| args: ["while true; do nvidia-smi -L; sleep 5; done"] | |
| resources: | |
| limits: | |
| nvidia.com/gpu: 1 | |
| EOF | |
| # deploy the test pod | |
| kubectl apply -f nvidia-smi-loop.yaml | |
| # back in node-shell, trigger the issue | |
| systemctl daemon-reload | |
| kubectl logs -f cuda-nvidia-smi-loop | |
| # GPU 0: NVIDIA A100 80GB PCIe (UUID: GPU-0451fe54-e0a1-36c5-eeb5-19025f49663e) | |
| # GPU 0: NVIDIA A100 80GB PCIe (UUID: GPU-0451fe54-e0a1-36c5-eeb5-19025f49663e) | |
| # GPU 0: NVIDIA A100 80GB PCIe (UUID: GPU-0451fe54-e0a1-36c5-eeb5-19025f49663e) | |
| # GPU 0: NVIDIA A100 80GB PCIe (UUID: GPU-0451fe54-e0a1-36c5-eeb5-19025f49663e) | |
| # Failed to initialize NVML: Unknown Error | |
| # Failed to initialize NVML: Unknown Error | |
| # output of nvidia-ctk was notably different on first/second repros | |
| # first attempt created /dev/char/234:foo and friends | |
| # second attempt created /dev/char/235:foo and friends | |
| # INFO[0000] Creating link /dev/char/195:254 => /dev/nvidia-modeset | |
| # INFO[0000] Creating link /dev/char/195:255 => /dev/nvidiactl | |
| # INFO[0000] Creating link /dev/char/509:0 => /dev/nvidia-uvm | |
| # INFO[0000] Creating link /dev/char/509:1 => /dev/nvidia-uvm-tools | |
| # INFO[0000] Creating link /dev/char/234:1 => /dev/nvidia-caps/nvidia-cap1 | |
| # INFO[0000] Creating link /dev/char/234:2 => /dev/nvidia-caps/nvidia-cap2 | |
| # INFO[0000] Creating link /dev/char/195:0 => /dev/nvidia0 | |
| # INFO[0000] Creating link /dev/char/234:3 => /dev/nvidia-caps/nvidia-cap3 | |
| # INFO[0000] Creating link /dev/char/234:4 => /dev/nvidia-caps/nvidia-cap4 | |
| # INFO[0000] Creating link /dev/char/234:5 => /dev/nvidia-caps/nvidia-cap5 | |
| # INFO[0000] Creating link /dev/char/234:6 => /dev/nvidia-caps/nvidia-cap6 | |
| # INFO[0000] Creating link /dev/char/234:7 => /dev/nvidia-caps/nvidia-cap7 | |
| # INFO[0000] Creating link /dev/char/234:8 => /dev/nvidia-caps/nvidia-cap8 | |
| # INFO[0000] Creating link /dev/char/234:9 => /dev/nvidia-caps/nvidia-cap9 | |
| # INFO[0000] Creating link /dev/char/234:10 => /dev/nvidia-caps/nvidia-cap10 | |
| # INFO[0000] Creating link /dev/char/234:11 => /dev/nvidia-caps/nvidia-cap11 | |
| # INFO[0000] Creating link /dev/char/234:12 => /dev/nvidia-caps/nvidia-cap12 | |
| # INFO[0000] Creating link /dev/char/234:13 => /dev/nvidia-caps/nvidia-cap13 | |
| # INFO[0000] Creating link /dev/char/234:14 => /dev/nvidia-caps/nvidia-cap14 | |
| # INFO[0000] Creating link /dev/char/234:15 => /dev/nvidia-caps/nvidia-cap15 | |
| # INFO[0000] Creating link /dev/char/234:16 => /dev/nvidia-caps/nvidia-cap16 | |
| # INFO[0000] Creating link /dev/char/234:17 => /dev/nvidia-caps/nvidia-cap17 | |
| # INFO[0000] Creating link /dev/char/234:18 => /dev/nvidia-caps/nvidia-cap18 | |
| # INFO[0000] Creating link /dev/char/234:19 => /dev/nvidia-caps/nvidia-cap19 | |
| # INFO[0000] Creating link /dev/char/234:20 => /dev/nvidia-caps/nvidia-cap20 | |
| # INFO[0000] Creating link /dev/char/234:21 => /dev/nvidia-caps/nvidia-cap21 | |
| # INFO[0000] Creating link /dev/char/234:22 => /dev/nvidia-caps/nvidia-cap22 | |
| # INFO[0000] Creating link /dev/char/234:23 => /dev/nvidia-caps/nvidia-cap23 | |
| # INFO[0000] Creating link /dev/char/234:24 => /dev/nvidia-caps/nvidia-cap24 | |
| # INFO[0000] Creating link /dev/char/234:25 => /dev/nvidia-caps/nvidia-cap25 | |
| # INFO[0000] Creating link /dev/char/234:26 => /dev/nvidia-caps/nvidia-cap26 | |
| # INFO[0000] Creating link /dev/char/234:27 => /dev/nvidia-caps/nvidia-cap27 | |
| # INFO[0000] Creating link /dev/char/234:28 => /dev/nvidia-caps/nvidia-cap28 | |
| # INFO[0000] Creating link /dev/char/234:29 => /dev/nvidia-caps/nvidia-cap29 | |
| # INFO[0000] Creating link /dev/char/234:30 => /dev/nvidia-caps/nvidia-cap30 | |
| # INFO[0000] Creating link /dev/char/234:31 => /dev/nvidia-caps/nvidia-cap31 | |
| # INFO[0000] Creating link /dev/char/234:32 => /dev/nvidia-caps/nvidia-cap32 | |
| # INFO[0000] Creating link /dev/char/234:33 => /dev/nvidia-caps/nvidia-cap33 | |
| # INFO[0000] Creating link /dev/char/234:34 => /dev/nvidia-caps/nvidia-cap34 | |
| # INFO[0000] Creating link /dev/char/234:35 => /dev/nvidia-caps/nvidia-cap35 | |
| # INFO[0000] Creating link /dev/char/234:36 => /dev/nvidia-caps/nvidia-cap36 | |
| # INFO[0000] Creating link /dev/char/234:37 => /dev/nvidia-caps/nvidia-cap37 | |
| # INFO[0000] Creating link /dev/char/234:38 => /dev/nvidia-caps/nvidia-cap38 | |
| # INFO[0000] Creating link /dev/char/234:39 => /dev/nvidia-caps/nvidia-cap39 | |
| # INFO[0000] Creating link /dev/char/234:40 => /dev/nvidia-caps/nvidia-cap40 | |
| # INFO[0000] Creating link /dev/char/234:41 => /dev/nvidia-caps/nvidia-cap41 | |
| # INFO[0000] Creating link /dev/char/234:42 => /dev/nvidia-caps/nvidia-cap42 | |
| # INFO[0000] Creating link /dev/char/234:43 => /dev/nvidia-caps/nvidia-cap43 | |
| # INFO[0000] Creating link /dev/char/234:44 => /dev/nvidia-caps/nvidia-cap44 | |
| # INFO[0000] Creating link /dev/char/234:45 => /dev/nvidia-caps/nvidia-cap45 | |
| # INFO[0000] Creating link /dev/char/234:46 => /dev/nvidia-caps/nvidia-cap46 | |
| # INFO[0000] Creating link /dev/char/234:47 => /dev/nvidia-caps/nvidia-cap47 | |
| # INFO[0000] Creating link /dev/char/234:48 => /dev/nvidia-caps/nvidia-cap48 | |
| # INFO[0000] Creating link /dev/char/234:49 => /dev/nvidia-caps/nvidia-cap49 | |
| # INFO[0000] Creating link /dev/char/234:50 => /dev/nvidia-caps/nvidia-cap50 | |
| # INFO[0000] Creating link /dev/char/234:51 => /dev/nvidia-caps/nvidia-cap51 | |
| # INFO[0000] Creating link /dev/char/234:52 => /dev/nvidia-caps/nvidia-cap52 | |
| # INFO[0000] Creating link /dev/char/234:53 => /dev/nvidia-caps/nvidia-cap53 | |
| # INFO[0000] Creating link /dev/char/234:54 => /dev/nvidia-caps/nvidia-cap54 | |
| # INFO[0000] Creating link /dev/char/234:55 => /dev/nvidia-caps/nvidia-cap55 | |
| # INFO[0000] Creating link /dev/char/234:56 => /dev/nvidia-caps/nvidia-cap56 | |
| # INFO[0000] Creating link /dev/char/234:57 => /dev/nvidia-caps/nvidia-cap57 | |
| # INFO[0000] Creating link /dev/char/234:58 => /dev/nvidia-caps/nvidia-cap58 | |
| # INFO[0000] Creating link /dev/char/234:59 => /dev/nvidia-caps/nvidia-cap59 | |
| # INFO[0000] Creating link /dev/char/234:60 => /dev/nvidia-caps/nvidia-cap60 | |
| # INFO[0000] Creating link /dev/char/234:61 => /dev/nvidia-caps/nvidia-cap61 | |
| # INFO[0000] Creating link /dev/char/234:62 => /dev/nvidia-caps/nvidia-cap62 | |
| # INFO[0000] Creating link /dev/char/234:63 => /dev/nvidia-caps/nvidia-cap63 | |
| # INFO[0000] Creating link /dev/char/234:64 => /dev/nvidia-caps/nvidia-cap64 | |
| # INFO[0000] Creating link /dev/char/234:65 => /dev/nvidia-caps/nvidia-cap65 | |
| # INFO[0000] Creating link /dev/char/234:66 => /dev/nvidia-caps/nvidia-cap66 | |
| # INFO[0000] Creating link /dev/char/234:67 => /dev/nvidia-caps/nvidia-cap67 | |
| # INFO[0000] Creating link /dev/char/234:68 => /dev/nvidia-caps/nvidia-cap68 | |
| # INFO[0000] Creating link /dev/char/234:69 => /dev/nvidia-caps/nvidia-cap69 | |
| # INFO[0000] Creating link /dev/char/234:70 => /dev/nvidia-caps/nvidia-cap70 | |
| # INFO[0000] Creating link /dev/char/234:71 => /dev/nvidia-caps/nvidia-cap71 | |
| # INFO[0000] Creating link /dev/char/234:72 => /dev/nvidia-caps/nvidia-cap72 | |
| # INFO[0000] Creating link /dev/char/234:73 => /dev/nvidia-caps/nvidia-cap73 | |
| # INFO[0000] Creating link /dev/char/234:74 => /dev/nvidia-caps/nvidia-cap74 | |
| # INFO[0000] Creating link /dev/char/234:75 => /dev/nvidia-caps/nvidia-cap75 | |
| # INFO[0000] Creating link /dev/char/234:76 => /dev/nvidia-caps/nvidia-cap76 | |
| # INFO[0000] Creating link /dev/char/234:77 => /dev/nvidia-caps/nvidia-cap77 | |
| # INFO[0000] Creating link /dev/char/234:78 => /dev/nvidia-caps/nvidia-cap78 | |
| # INFO[0000] Creating link /dev/char/234:79 => /dev/nvidia-caps/nvidia-cap79 | |
| # INFO[0000] Creating link /dev/char/234:80 => /dev/nvidia-caps/nvidia-cap80 | |
| # INFO[0000] Creating link /dev/char/234:81 => /dev/nvidia-caps/nvidia-cap81 | |
| # INFO[0000] Creating link /dev/char/234:82 => /dev/nvidia-caps/nvidia-cap82 | |
| # INFO[0000] Creating link /dev/char/234:83 => /dev/nvidia-caps/nvidia-cap83 | |
| # INFO[0000] Creating link /dev/char/234:84 => /dev/nvidia-caps/nvidia-cap84 | |
| # INFO[0000] Creating link /dev/char/234:85 => /dev/nvidia-caps/nvidia-cap85 | |
| # INFO[0000] Creating link /dev/char/234:86 => /dev/nvidia-caps/nvidia-cap86 | |
| # INFO[0000] Creating link /dev/char/234:87 => /dev/nvidia-caps/nvidia-cap87 | |
| # INFO[0000] Creating link /dev/char/234:88 => /dev/nvidia-caps/nvidia-cap88 | |
| # INFO[0000] Creating link /dev/char/234:89 => /dev/nvidia-caps/nvidia-cap89 | |
| # INFO[0000] Creating link /dev/char/234:90 => /dev/nvidia-caps/nvidia-cap90 | |
| # INFO[0000] Creating link /dev/char/234:91 => /dev/nvidia-caps/nvidia-cap91 | |
| # INFO[0000] Creating link /dev/char/234:92 => /dev/nvidia-caps/nvidia-cap92 | |
| # INFO[0000] Creating link /dev/char/234:93 => /dev/nvidia-caps/nvidia-cap93 | |
| # INFO[0000] Creating link /dev/char/234:94 => /dev/nvidia-caps/nvidia-cap94 | |
| # INFO[0000] Creating link /dev/char/234:95 => /dev/nvidia-caps/nvidia-cap95 | |
| # INFO[0000] Creating link /dev/char/234:96 => /dev/nvidia-caps/nvidia-cap96 | |
| # INFO[0000] Creating link /dev/char/234:97 => /dev/nvidia-caps/nvidia-cap97 | |
| # INFO[0000] Creating link /dev/char/234:98 => /dev/nvidia-caps/nvidia-cap98 | |
| # INFO[0000] Creating link /dev/char/234:99 => /dev/nvidia-caps/nvidia-cap99 | |
| # INFO[0000] Creating link /dev/char/234:100 => /dev/nvidia-caps/nvidia-cap100 | |
| # INFO[0000] Creating link /dev/char/234:101 => /dev/nvidia-caps/nvidia-cap101 | |
| # INFO[0000] Creating link /dev/char/234:102 => /dev/nvidia-caps/nvidia-cap102 | |
| # INFO[0000] Creating link /dev/char/234:103 => /dev/nvidia-caps/nvidia-cap103 | |
| # INFO[0000] Creating link /dev/char/234:104 => /dev/nvidia-caps/nvidia-cap104 | |
| # INFO[0000] Creating link /dev/char/234:105 => /dev/nvidia-caps/nvidia-cap105 | |
| # INFO[0000] Creating link /dev/char/234:106 => /dev/nvidia-caps/nvidia-cap106 | |
| # INFO[0000] Creating link /dev/char/234:107 => /dev/nvidia-caps/nvidia-cap107 | |
| # INFO[0000] Creating link /dev/char/234:108 => /dev/nvidia-caps/nvidia-cap108 | |
| # INFO[0000] Creating link /dev/char/234:109 => /dev/nvidia-caps/nvidia-cap109 | |
| # INFO[0000] Creating link /dev/char/234:110 => /dev/nvidia-caps/nvidia-cap110 | |
| # INFO[0000] Creating link /dev/char/234:111 => /dev/nvidia-caps/nvidia-cap111 | |
| # INFO[0000] Creating link /dev/char/234:112 => /dev/nvidia-caps/nvidia-cap112 | |
| # INFO[0000] Creating link /dev/char/234:113 => /dev/nvidia-caps/nvidia-cap113 | |
| # INFO[0000] Creating link /dev/char/234:114 => /dev/nvidia-caps/nvidia-cap114 | |
| # INFO[0000] Creating link /dev/char/234:115 => /dev/nvidia-caps/nvidia-cap115 | |
| # INFO[0000] Creating link /dev/char/234:116 => /dev/nvidia-caps/nvidia-cap116 | |
| # INFO[0000] Creating link /dev/char/234:117 => /dev/nvidia-caps/nvidia-cap117 | |
| # INFO[0000] Creating link /dev/char/234:118 => /dev/nvidia-caps/nvidia-cap118 | |
| # INFO[0000] Creating link /dev/char/234:119 => /dev/nvidia-caps/nvidia-cap119 | |
| # INFO[0000] Creating link /dev/char/234:120 => /dev/nvidia-caps/nvidia-cap120 | |
| # INFO[0000] Creating link /dev/char/234:121 => /dev/nvidia-caps/nvidia-cap121 | |
| # INFO[0000] Creating link /dev/char/234:122 => /dev/nvidia-caps/nvidia-cap122 | |
| # INFO[0000] Creating link /dev/char/234:123 => /dev/nvidia-caps/nvidia-cap123 | |
| # INFO[0000] Creating link /dev/char/234:124 => /dev/nvidia-caps/nvidia-cap124 | |
| # INFO[0000] Creating link /dev/char/234:125 => /dev/nvidia-caps/nvidia-cap125 | |
| # INFO[0000] Creating link /dev/char/234:126 => /dev/nvidia-caps/nvidia-cap126 | |
| # INFO[0000] Creating link /dev/char/234:127 => /dev/nvidia-caps/nvidia-cap127 | |
| # INFO[0000] Creating link /dev/char/234:128 => /dev/nvidia-caps/nvidia-cap128 | |
| # INFO[0000] Creating link /dev/char/234:129 => /dev/nvidia-caps/nvidia-cap129 | |
| # INFO[0000] Creating link /dev/char/234:130 => /dev/nvidia-caps/nvidia-cap130 | |
| # INFO[0000] Creating link /dev/char/234:131 => /dev/nvidia-caps/nvidia-cap131 | |
| # INFO[0000] Creating link /dev/char/234:132 => /dev/nvidia-caps/nvidia-cap132 | |
| # INFO[0000] Creating link /dev/char/234:133 => /dev/nvidia-caps/nvidia-cap133 | |
| # INFO[0000] Creating link /dev/char/234:134 => /dev/nvidia-caps/nvidia-cap134 | |
| # INFO[0000] Creating link /dev/char/234:135 => /dev/nvidia-caps/nvidia-cap135 | |
| # INFO[0000] Creating link /dev/char/234:136 => /dev/nvidia-caps/nvidia-cap136 | |
| # INFO[0000] Creating link /dev/char/234:137 => /dev/nvidia-caps/nvidia-cap137 | |
| # second run | |
| root@aks-nca100-36400834-vmss000000:~# nvidia-ctk system create-dev-char-symlinks --create-all | |
| # INFO[0000] Creating link /dev/char/195:254 => /dev/nvidia-modeset | |
| # INFO[0000] Creating link /dev/char/195:255 => /dev/nvidiactl | |
| # INFO[0000] Creating link /dev/char/509:0 => /dev/nvidia-uvm | |
| # INFO[0000] Creating link /dev/char/509:1 => /dev/nvidia-uvm-tools | |
| # INFO[0000] Creating link /dev/char/235:1 => /dev/nvidia-caps/nvidia-cap1 | |
| # INFO[0000] Creating link /dev/char/235:2 => /dev/nvidia-caps/nvidia-cap2 | |
| # INFO[0000] Creating link /dev/char/195:0 => /dev/nvidia0 | |
| # INFO[0000] Creating link /dev/char/235:3 => /dev/nvidia-caps/nvidia-cap3 | |
| # INFO[0000] Creating link /dev/char/235:4 => /dev/nvidia-caps/nvidia-cap4 | |
| # INFO[0000] Creating link /dev/char/235:5 => /dev/nvidia-caps/nvidia-cap5 | |
| # INFO[0000] Creating link /dev/char/235:6 => /dev/nvidia-caps/nvidia-cap6 | |
| # INFO[0000] Creating link /dev/char/235:7 => /dev/nvidia-caps/nvidia-cap7 | |
| # INFO[0000] Creating link /dev/char/235:8 => /dev/nvidia-caps/nvidia-cap8 | |
| # INFO[0000] Creating link /dev/char/235:9 => /dev/nvidia-caps/nvidia-cap9 | |
| # INFO[0000] Creating link /dev/char/235:10 => /dev/nvidia-caps/nvidia-cap10 | |
| # INFO[0000] Creating link /dev/char/235:11 => /dev/nvidia-caps/nvidia-cap11 | |
| # INFO[0000] Creating link /dev/char/235:12 => /dev/nvidia-caps/nvidia-cap12 | |
| # INFO[0000] Creating link /dev/char/235:13 => /dev/nvidia-caps/nvidia-cap13 | |
| # INFO[0000] Creating link /dev/char/235:14 => /dev/nvidia-caps/nvidia-cap14 | |
| # INFO[0000] Creating link /dev/char/235:15 => /dev/nvidia-caps/nvidia-cap15 | |
| # INFO[0000] Creating link /dev/char/235:16 => /dev/nvidia-caps/nvidia-cap16 | |
| # INFO[0000] Creating link /dev/char/235:17 => /dev/nvidia-caps/nvidia-cap17 | |
| # INFO[0000] Creating link /dev/char/235:18 => /dev/nvidia-caps/nvidia-cap18 | |
| # INFO[0000] Creating link /dev/char/235:19 => /dev/nvidia-caps/nvidia-cap19 | |
| # INFO[0000] Creating link /dev/char/235:20 => /dev/nvidia-caps/nvidia-cap20 | |
| # INFO[0000] Creating link /dev/char/235:21 => /dev/nvidia-caps/nvidia-cap21 | |
| # INFO[0000] Creating link /dev/char/235:22 => /dev/nvidia-caps/nvidia-cap22 | |
| # INFO[0000] Creating link /dev/char/235:23 => /dev/nvidia-caps/nvidia-cap23 | |
| # INFO[0000] Creating link /dev/char/235:24 => /dev/nvidia-caps/nvidia-cap24 | |
| # INFO[0000] Creating link /dev/char/235:25 => /dev/nvidia-caps/nvidia-cap25 | |
| # INFO[0000] Creating link /dev/char/235:26 => /dev/nvidia-caps/nvidia-cap26 | |
| # INFO[0000] Creating link /dev/char/235:27 => /dev/nvidia-caps/nvidia-cap27 | |
| # INFO[0000] Creating link /dev/char/235:28 => /dev/nvidia-caps/nvidia-cap28 | |
| # INFO[0000] Creating link /dev/char/235:29 => /dev/nvidia-caps/nvidia-cap29 | |
| # INFO[0000] Creating link /dev/char/235:30 => /dev/nvidia-caps/nvidia-cap30 | |
| # INFO[0000] Creating link /dev/char/235:31 => /dev/nvidia-caps/nvidia-cap31 | |
| # INFO[0000] Creating link /dev/char/235:32 => /dev/nvidia-caps/nvidia-cap32 | |
| # INFO[0000] Creating link /dev/char/235:33 => /dev/nvidia-caps/nvidia-cap33 | |
| # INFO[0000] Creating link /dev/char/235:34 => /dev/nvidia-caps/nvidia-cap34 | |
| # INFO[0000] Creating link /dev/char/235:35 => /dev/nvidia-caps/nvidia-cap35 | |
| # INFO[0000] Creating link /dev/char/235:36 => /dev/nvidia-caps/nvidia-cap36 | |
| # INFO[0000] Creating link /dev/char/235:37 => /dev/nvidia-caps/nvidia-cap37 | |
| # INFO[0000] Creating link /dev/char/235:38 => /dev/nvidia-caps/nvidia-cap38 | |
| # INFO[0000] Creating link /dev/char/235:39 => /dev/nvidia-caps/nvidia-cap39 | |
| # INFO[0000] Creating link /dev/char/235:40 => /dev/nvidia-caps/nvidia-cap40 | |
| # INFO[0000] Creating link /dev/char/235:41 => /dev/nvidia-caps/nvidia-cap41 | |
| # INFO[0000] Creating link /dev/char/235:42 => /dev/nvidia-caps/nvidia-cap42 | |
| # INFO[0000] Creating link /dev/char/235:43 => /dev/nvidia-caps/nvidia-cap43 | |
| # INFO[0000] Creating link /dev/char/235:44 => /dev/nvidia-caps/nvidia-cap44 | |
| # INFO[0000] Creating link /dev/char/235:45 => /dev/nvidia-caps/nvidia-cap45 | |
| # INFO[0000] Creating link /dev/char/235:46 => /dev/nvidia-caps/nvidia-cap46 | |
| # INFO[0000] Creating link /dev/char/235:47 => /dev/nvidia-caps/nvidia-cap47 | |
| # INFO[0000] Creating link /dev/char/235:48 => /dev/nvidia-caps/nvidia-cap48 | |
| # INFO[0000] Creating link /dev/char/235:49 => /dev/nvidia-caps/nvidia-cap49 | |
| # INFO[0000] Creating link /dev/char/235:50 => /dev/nvidia-caps/nvidia-cap50 | |
| # INFO[0000] Creating link /dev/char/235:51 => /dev/nvidia-caps/nvidia-cap51 | |
| # INFO[0000] Creating link /dev/char/235:52 => /dev/nvidia-caps/nvidia-cap52 | |
| # INFO[0000] Creating link /dev/char/235:53 => /dev/nvidia-caps/nvidia-cap53 | |
| # INFO[0000] Creating link /dev/char/235:54 => /dev/nvidia-caps/nvidia-cap54 | |
| # INFO[0000] Creating link /dev/char/235:55 => /dev/nvidia-caps/nvidia-cap55 | |
| # INFO[0000] Creating link /dev/char/235:56 => /dev/nvidia-caps/nvidia-cap56 | |
| # INFO[0000] Creating link /dev/char/235:57 => /dev/nvidia-caps/nvidia-cap57 | |
| # INFO[0000] Creating link /dev/char/235:58 => /dev/nvidia-caps/nvidia-cap58 | |
| # INFO[0000] Creating link /dev/char/235:59 => /dev/nvidia-caps/nvidia-cap59 | |
| # INFO[0000] Creating link /dev/char/235:60 => /dev/nvidia-caps/nvidia-cap60 | |
| # INFO[0000] Creating link /dev/char/235:61 => /dev/nvidia-caps/nvidia-cap61 | |
| # INFO[0000] Creating link /dev/char/235:62 => /dev/nvidia-caps/nvidia-cap62 | |
| # INFO[0000] Creating link /dev/char/235:63 => /dev/nvidia-caps/nvidia-cap63 | |
| # INFO[0000] Creating link /dev/char/235:64 => /dev/nvidia-caps/nvidia-cap64 | |
| # INFO[0000] Creating link /dev/char/235:65 => /dev/nvidia-caps/nvidia-cap65 | |
| # INFO[0000] Creating link /dev/char/235:66 => /dev/nvidia-caps/nvidia-cap66 | |
| # INFO[0000] Creating link /dev/char/235:67 => /dev/nvidia-caps/nvidia-cap67 | |
| # INFO[0000] Creating link /dev/char/235:68 => /dev/nvidia-caps/nvidia-cap68 | |
| # INFO[0000] Creating link /dev/char/235:69 => /dev/nvidia-caps/nvidia-cap69 | |
| # INFO[0000] Creating link /dev/char/235:70 => /dev/nvidia-caps/nvidia-cap70 | |
| # INFO[0000] Creating link /dev/char/235:71 => /dev/nvidia-caps/nvidia-cap71 | |
| # INFO[0000] Creating link /dev/char/235:72 => /dev/nvidia-caps/nvidia-cap72 | |
| # INFO[0000] Creating link /dev/char/235:73 => /dev/nvidia-caps/nvidia-cap73 | |
| # INFO[0000] Creating link /dev/char/235:74 => /dev/nvidia-caps/nvidia-cap74 | |
| # INFO[0000] Creating link /dev/char/235:75 => /dev/nvidia-caps/nvidia-cap75 | |
| # INFO[0000] Creating link /dev/char/235:76 => /dev/nvidia-caps/nvidia-cap76 | |
| # INFO[0000] Creating link /dev/char/235:77 => /dev/nvidia-caps/nvidia-cap77 | |
| # INFO[0000] Creating link /dev/char/235:78 => /dev/nvidia-caps/nvidia-cap78 | |
| # INFO[0000] Creating link /dev/char/235:79 => /dev/nvidia-caps/nvidia-cap79 | |
| # INFO[0000] Creating link /dev/char/235:80 => /dev/nvidia-caps/nvidia-cap80 | |
| # INFO[0000] Creating link /dev/char/235:81 => /dev/nvidia-caps/nvidia-cap81 | |
| # INFO[0000] Creating link /dev/char/235:82 => /dev/nvidia-caps/nvidia-cap82 | |
| # INFO[0000] Creating link /dev/char/235:83 => /dev/nvidia-caps/nvidia-cap83 | |
| # INFO[0000] Creating link /dev/char/235:84 => /dev/nvidia-caps/nvidia-cap84 | |
| # INFO[0000] Creating link /dev/char/235:85 => /dev/nvidia-caps/nvidia-cap85 | |
| # INFO[0000] Creating link /dev/char/235:86 => /dev/nvidia-caps/nvidia-cap86 | |
| # INFO[0000] Creating link /dev/char/235:87 => /dev/nvidia-caps/nvidia-cap87 | |
| # INFO[0000] Creating link /dev/char/235:88 => /dev/nvidia-caps/nvidia-cap88 | |
| # INFO[0000] Creating link /dev/char/235:89 => /dev/nvidia-caps/nvidia-cap89 | |
| # INFO[0000] Creating link /dev/char/235:90 => /dev/nvidia-caps/nvidia-cap90 | |
| # INFO[0000] Creating link /dev/char/235:91 => /dev/nvidia-caps/nvidia-cap91 | |
| # INFO[0000] Creating link /dev/char/235:92 => /dev/nvidia-caps/nvidia-cap92 | |
| # INFO[0000] Creating link /dev/char/235:93 => /dev/nvidia-caps/nvidia-cap93 | |
| # INFO[0000] Creating link /dev/char/235:94 => /dev/nvidia-caps/nvidia-cap94 | |
| # INFO[0000] Creating link /dev/char/235:95 => /dev/nvidia-caps/nvidia-cap95 | |
| # INFO[0000] Creating link /dev/char/235:96 => /dev/nvidia-caps/nvidia-cap96 | |
| # INFO[0000] Creating link /dev/char/235:97 => /dev/nvidia-caps/nvidia-cap97 | |
| # INFO[0000] Creating link /dev/char/235:98 => /dev/nvidia-caps/nvidia-cap98 | |
| # INFO[0000] Creating link /dev/char/235:99 => /dev/nvidia-caps/nvidia-cap99 | |
| # INFO[0000] Creating link /dev/char/235:100 => /dev/nvidia-caps/nvidia-cap100 | |
| # INFO[0000] Creating link /dev/char/235:101 => /dev/nvidia-caps/nvidia-cap101 | |
| # INFO[0000] Creating link /dev/char/235:102 => /dev/nvidia-caps/nvidia-cap102 | |
| # INFO[0000] Creating link /dev/char/235:103 => /dev/nvidia-caps/nvidia-cap103 | |
| # INFO[0000] Creating link /dev/char/235:104 => /dev/nvidia-caps/nvidia-cap104 | |
| # INFO[0000] Creating link /dev/char/235:105 => /dev/nvidia-caps/nvidia-cap105 | |
| # INFO[0000] Creating link /dev/char/235:106 => /dev/nvidia-caps/nvidia-cap106 | |
| # INFO[0000] Creating link /dev/char/235:107 => /dev/nvidia-caps/nvidia-cap107 | |
| # INFO[0000] Creating link /dev/char/235:108 => /dev/nvidia-caps/nvidia-cap108 | |
| # INFO[0000] Creating link /dev/char/235:109 => /dev/nvidia-caps/nvidia-cap109 | |
| # INFO[0000] Creating link /dev/char/235:110 => /dev/nvidia-caps/nvidia-cap110 | |
| # INFO[0000] Creating link /dev/char/235:111 => /dev/nvidia-caps/nvidia-cap111 | |
| # INFO[0000] Creating link /dev/char/235:112 => /dev/nvidia-caps/nvidia-cap112 | |
| # INFO[0000] Creating link /dev/char/235:113 => /dev/nvidia-caps/nvidia-cap113 | |
| # INFO[0000] Creating link /dev/char/235:114 => /dev/nvidia-caps/nvidia-cap114 | |
| # INFO[0000] Creating link /dev/char/235:115 => /dev/nvidia-caps/nvidia-cap115 | |
| # INFO[0000] Creating link /dev/char/235:116 => /dev/nvidia-caps/nvidia-cap116 | |
| # INFO[0000] Creating link /dev/char/235:117 => /dev/nvidia-caps/nvidia-cap117 | |
| # INFO[0000] Creating link /dev/char/235:118 => /dev/nvidia-caps/nvidia-cap118 | |
| # INFO[0000] Creating link /dev/char/235:119 => /dev/nvidia-caps/nvidia-cap119 | |
| # INFO[0000] Creating link /dev/char/235:120 => /dev/nvidia-caps/nvidia-cap120 | |
| # INFO[0000] Creating link /dev/char/235:121 => /dev/nvidia-caps/nvidia-cap121 | |
| # INFO[0000] Creating link /dev/char/235:122 => /dev/nvidia-caps/nvidia-cap122 | |
| # INFO[0000] Creating link /dev/char/235:123 => /dev/nvidia-caps/nvidia-cap123 | |
| # INFO[0000] Creating link /dev/char/235:124 => /dev/nvidia-caps/nvidia-cap124 | |
| # INFO[0000] Creating link /dev/char/235:125 => /dev/nvidia-caps/nvidia-cap125 | |
| # INFO[0000] Creating link /dev/char/235:126 => /dev/nvidia-caps/nvidia-cap126 | |
| # INFO[0000] Creating link /dev/char/235:127 => /dev/nvidia-caps/nvidia-cap127 | |
| # INFO[0000] Creating link /dev/char/235:128 => /dev/nvidia-caps/nvidia-cap128 | |
| # INFO[0000] Creating link /dev/char/235:129 => /dev/nvidia-caps/nvidia-cap129 | |
| # INFO[0000] Creating link /dev/char/235:130 => /dev/nvidia-caps/nvidia-cap130 | |
| # INFO[0000] Creating link /dev/char/235:131 => /dev/nvidia-caps/nvidia-cap131 | |
| # INFO[0000] Creating link /dev/char/235:132 => /dev/nvidia-caps/nvidia-cap132 | |
| # INFO[0000] Creating link /dev/char/235:133 => /dev/nvidia-caps/nvidia-cap133 | |
| # INFO[0000] Creating link /dev/char/235:134 => /dev/nvidia-caps/nvidia-cap134 | |
| # INFO[0000] Creating link /dev/char/235:135 => /dev/nvidia-caps/nvidia-cap135 | |
| # INFO[0000] Creating link /dev/char/235:136 => /dev/nvidia-caps/nvidia-cap136 | |
| # INFO[0000] Creating link /dev/char/235:137 => /dev/nvidia-caps/nvidia-cap137 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment