Skip to content

Instantly share code, notes, and snippets.

@tormath1

tormath1/debug.md

Last active July 24, 2024 15:08
Show Gist options
  • Save tormath1/bd57b210343f5b1e93aa8ed5ada36d40 to your computer and use it in GitHub Desktop.
Save tormath1/bd57b210343f5b1e93aa8ed5ada36d40 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
debug.md 
$ sudo crictl images
IMAGE                                     TAG                 IMAGE ID            SIZE
registry.k8s.io/coredns/coredns           v1.11.1             cbb01a7bd410d       18.2MB
registry.k8s.io/etcd                      3.5.12-0            3861cfcd7c04c       57.2MB
registry.k8s.io/kube-apiserver            v1.29.5             b36112597a5f1       35.2MB
registry.k8s.io/kube-controller-manager   v1.29.5             2242ad7f7c41a       33.6MB
registry.k8s.io/kube-proxy                v1.29.5             2019bbea5542a       28.4MB
registry.k8s.io/kube-scheduler            v1.29.5             e579eb50f57be       18.7MB
registry.k8s.io/pause                     3.8                 4873874c08efc       311kB
registry.k8s.io/pause                     3.9                 e6f1816883972       322kB
$ sudo crictl ps  -a
CONTAINER           IMAGE               CREATED             STATE               NAME                      ATTEMPT             POD ID              POD
7ff6af2379d97       3861cfcd7c04c       8 minutes ago       Running             etcd                      0                   2afed56a51fde       etcd-test-cluster-control-plane-vfzjb
177529b72bdde       b36112597a5f1       8 minutes ago       Running             kube-apiserver            0                   a8dcf282c7b1f       kube-apiserver-test-cluster-control-plane-vfzjb
d96591c4353ec       2242ad7f7c41a       8 minutes ago       Running             kube-controller-manager   0                   a078b137ecbe9       kube-controller-manager-test-cluster-control-plane-vfzjb
0604d4eb15b12       e579eb50f57be       8 minutes ago       Running             kube-scheduler            0                   456b7fa6cd949       kube-scheduler-test-cluster-control-plane-vfzjb
$ sudo cat /etc/kubeadm.yml
---
apiServer:
  extraArgs:
    cloud-provider: external
apiVersion: kubeadm.k8s.io/v1beta3
clusterName: test-cluster
controlPlaneEndpoint: 172.232.16.156:6443
controllerManager:
  extraArgs:
    cloud-provider: external
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd_data/etcd
    extraArgs:
      quota-backend-bytes: "8589934592"
kind: ClusterConfiguration
kubernetesVersion: v1.29.5
networking:
  podSubnet: 10.192.0.0/10
scheduler: {}

---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint: {}
nodeRegistration:
  imagePullPolicy: IfNotPresent
  kubeletExtraArgs:
    cloud-provider: external
    provider-id: linode:///60752639
  name: test-cluster-control-plane-vfzjb
  taints: null
Raw
etcd.log
$ kubectl --kubeconfig ./test-cluster.kubeconfig logs -f etcd-test-cluster-control-plane-xwsf8 -n kube-system -f
{"level":"warn","ts":"2024-07-24T15:07:24.251583Z","caller":"embed/config.go:679","msg":"Running http and grpc server on single port. This is not recommended for production."}
{"level":"info","ts":"2024-07-24T15:07:24.251706Z","caller":"etcdmain/etcd.go:73","msg":"Running: ","args":["etcd","--advertise-client-urls=https://192.168.129.12:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd_data/etcd","--experimental-initial-corrupt-check=true","--experimental-watch-progress-notify-interval=5s","--initial-advertise-peer-urls=https://192.168.129.12:2380","--initial-cluster=test-cluster-control-plane-xwsf8=https://192.168.129.12:2380,test-cluster-control-plane-6jfk7=https://192.168.151.231:2380","--initial-cluster-state=existing","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://192.168.129.12:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://192.168.129.12:2380","--name=test-cluster-control-plane-xwsf8","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--quota-backend-bytes=8589934592","--snapshot-count=10000","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt"]}
{"level":"info","ts":"2024-07-24T15:07:24.251846Z","caller":"etcdmain/etcd.go:116","msg":"server has been already initialized","data-dir":"/var/lib/etcd_data/etcd","dir-type":"member"}
{"level":"warn","ts":"2024-07-24T15:07:24.251876Z","caller":"embed/config.go:679","msg":"Running http and grpc server on single port. This is not recommended for production."}
{"level":"info","ts":"2024-07-24T15:07:24.251887Z","caller":"embed/etcd.go:127","msg":"configuring peer listeners","listen-peer-urls":["https://192.168.129.12:2380"]}
{"level":"info","ts":"2024-07-24T15:07:24.251907Z","caller":"embed/etcd.go:494","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]}
{"level":"info","ts":"2024-07-24T15:07:24.253301Z","caller":"embed/etcd.go:135","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://192.168.129.12:2379"]}
{"level":"info","ts":"2024-07-24T15:07:24.253411Z","caller":"embed/etcd.go:308","msg":"starting an etcd server","etcd-version":"3.5.12","git-sha":"e7b3bb6cc","go-version":"go1.20.13","go-os":"linux","go-arch":"amd64","max-cpu-set":2,"max-cpu-available":2,"member-initialized":false,"name":"test-cluster-control-plane-xwsf8","data-dir":"/var/lib/etcd_data/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd_data/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://192.168.129.12:2380"],"listen-peer-urls":["https://192.168.129.12:2380"],"advertise-client-urls":["https://192.168.129.12:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://192.168.129.12:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"cors":["*"],"host-whitelist":["*"],"initial-cluster":"test-cluster-control-plane-6jfk7=https://192.168.151.231:2380,test-cluster-control-plane-xwsf8=https://192.168.129.12:2380","initial-cluster-state":"existing","initial-cluster-token":"etcd-cluster","quota-backend-bytes":8589934592,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"initial-corrupt-check":true,"corrupt-check-time-interval":"0s","compact-check-time-enabled":false,"compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","downgrade-check-interval":"5s"}
{"level":"info","ts":"2024-07-24T15:07:24.257164Z","caller":"etcdserver/backend.go:81","msg":"opened backend db","path":"/var/lib/etcd_data/etcd/member/snap/db","took":"3.346802ms"}
{"level":"warn","ts":"2024-07-24T15:07:24.441739Z","caller":"etcdserver/cluster_util.go:82","msg":"failed to get cluster response","address":"https://192.168.151.231:2380/members","error":"Get \"https://192.168.151.231:2380/members\": EOF"}
{"level":"info","ts":"2024-07-24T15:07:24.443468Z","caller":"embed/etcd.go:375","msg":"closing etcd server","name":"test-cluster-control-plane-xwsf8","data-dir":"/var/lib/etcd_data/etcd","advertise-peer-urls":["https://192.168.129.12:2380"],"advertise-client-urls":["https://192.168.129.12:2379"]}
{"level":"warn","ts":"2024-07-24T15:07:24.451458Z","caller":"embed/config_logging.go:160","msg":"rejected connection","remote-addr":"172.234.197.67:44392","server-name":"","ip-addresses":["192.168.151.231","127.0.0.1","::1"],"dns-names":["localhost","test-cluster-control-plane-6jfk7"],"error":"tls: \"172.234.197.67\" does not match any of DNSNames [\"localhost\" \"test-cluster-control-plane-6jfk7\"] (lookup test-cluster-control-plane-6jfk7 on 172.232.0.17:53: dial udp 172.232.0.17:53: operation was canceled)"}
{"level":"warn","ts":"2024-07-24T15:07:24.537374Z","caller":"embed/config_logging.go:160","msg":"rejected connection","remote-addr":"172.234.197.67:44404","server-name":"","ip-addresses":["192.168.151.231","127.0.0.1","::1"],"dns-names":["localhost","test-cluster-control-plane-6jfk7"],"error":"tls: \"172.234.197.67\" does not match any of DNSNames [\"localhost\" \"test-cluster-control-plane-6jfk7\"] (lookup test-cluster-control-plane-6jfk7 on 172.232.0.17:53: dial udp 172.232.0.17:53: operation was canceled)"}
{"level":"warn","ts":"2024-07-24T15:07:24.55056Z","caller":"embed/config_logging.go:160","msg":"rejected connection","remote-addr":"172.234.197.67:44386","server-name":"","ip-addresses":["192.168.151.231","127.0.0.1","::1"],"dns-names":["localhost","test-cluster-control-plane-6jfk7"],"error":"tls: \"172.234.197.67\" does not match any of DNSNames [\"localhost\" \"test-cluster-control-plane-6jfk7\"] (lookup test-cluster-control-plane-6jfk7 on 172.232.0.17:53: dial udp 172.232.0.17:53: operation was canceled)"}
{"level":"warn","ts":"2024-07-24T15:07:24.636927Z","caller":"embed/config_logging.go:160","msg":"rejected connection","remote-addr":"172.234.197.67:44414","server-name":"","ip-addresses":["192.168.151.231","127.0.0.1","::1"],"dns-names":["localhost","test-cluster-control-plane-6jfk7"],"error":"tls: \"172.234.197.67\" does not match any of DNSNames [\"localhost\" \"test-cluster-control-plane-6jfk7\"] (lookup test-cluster-control-plane-6jfk7 on 172.232.0.17:53: dial udp 172.232.0.17:53: operation was canceled)"}
{"level":"info","ts":"2024-07-24T15:07:24.637191Z","caller":"embed/etcd.go:377","msg":"closed etcd server","name":"test-cluster-control-plane-xwsf8","data-dir":"/var/lib/etcd_data/etcd","advertise-peer-urls":["https://192.168.129.12:2380"],"advertise-client-urls":["https://192.168.129.12:2379"]}
{"level":"fatal","ts":"2024-07-24T15:07:24.637372Z","caller":"etcdmain/etcd.go:204","msg":"discovery failed","error":"cannot fetch cluster info from peer urls: could not retrieve cluster information from the given URLs","stacktrace":"go.etcd.io/etcd/server/v3/etcdmain.startEtcdOrProxyV2\n\tgo.etcd.io/etcd/server/v3/etcdmain/etcd.go:204\ngo.etcd.io/etcd/server/v3/etcdmain.Main\n\tgo.etcd.io/etcd/server/v3/etcdmain/main.go:40\nmain.main\n\tgo.etcd.io/etcd/server/v3/main.go:31\nruntime.main\n\truntime/proc.go:250"}
Raw
test-cluster.yaml
apiVersion: v1
data:
cilium-policy.yaml: |-
apiVersion: "cilium.io/v2"
kind: CiliumClusterwideNetworkPolicy
metadata:
name: "default-cluster-policy"
spec:
description: "allow cluster intra cluster traffic"
endpointSelector: {}
ingress:
- fromEntities:
- cluster
- fromCIDR:
- 10.0.0.0/8
- 192.168.128.0/17
---
apiVersion: "cilium.io/v2"
kind: CiliumClusterwideNetworkPolicy
metadata:
name: "default-external-policy"
spec:
description: "allow api server traffic"
nodeSelector: {}
ingress:
- fromEntities:
- cluster
- fromCIDR:
- 10.0.0.0/8
- fromEntities:
- world
toPorts:
- ports:
- port: "6443"
kind: ConfigMap
metadata:
name: test-cluster-cilium-policy
namespace: default
---
apiVersion: v1
data:
cilium-policy.yaml: |-
apiVersion: "cilium.io/v2"
kind: CiliumClusterwideNetworkPolicy
metadata:
name: "allow-etcd-policy"
spec:
description: "allow etcd traffic"
nodeSelector: {}
ingress:
- fromEntities:
- world
toPorts:
- ports:
- port: "2379"
- port: "2380"
kind: ConfigMap
metadata:
name: test-cluster-etcd-cilium-policy
namespace: default
---
apiVersion: v1
kind: Secret
metadata:
labels:
clusterctl.cluster.x-k8s.io/move: "true"
name: test-cluster-credentials
namespace: default
stringData:
apiToken: redacted
---
apiVersion: v1
kind: Secret
metadata:
name: linode-test-cluster-crs-0
namespace: default
stringData:
linode-token-region.yaml: |-
kind: Secret
apiVersion: v1
metadata:
name: linode-token-region
namespace: kube-system
stringData:
apiToken: redacted
region: us-ord
type: addons.cluster.x-k8s.io/resource-set
---
apiVersion: addons.cluster.x-k8s.io/v1alpha1
kind: HelmChartProxy
metadata:
name: test-cluster-cilium
namespace: default
spec:
chartName: cilium
clusterSelector:
matchLabels:
cni: test-cluster-cilium
namespace: kube-system
options:
timeout: 5m
wait: true
waitForJobs: true
repoURL: https://helm.cilium.io/
valuesTemplate: |-
bgpControlPlane:
enabled: true
ipam:
mode: kubernetes
k8s:
requireIPv4PodCIDR: true
hubble:
relay:
enabled: true
ui:
enabled: true
version: 1.15.4
---
apiVersion: addons.cluster.x-k8s.io/v1alpha1
kind: HelmChartProxy
metadata:
name: test-cluster-csi-driver-linode
namespace: default
spec:
chartName: linode-blockstorage-csi-driver
clusterSelector:
matchLabels:
csi: test-cluster-linode
namespace: kube-system
options:
timeout: 5m
wait: true
waitForJobs: true
repoURL: https://linode.github.io/linode-blockstorage-csi-driver/
valuesTemplate: |
secretRef:
name: "linode-token-region"
apiTokenRef: "apiToken"
version: v0.7.0
---
apiVersion: addons.cluster.x-k8s.io/v1alpha1
kind: HelmChartProxy
metadata:
name: test-cluster-linode-cloud-controller-manager
namespace: default
spec:
chartName: ccm-linode
clusterSelector:
matchLabels:
ccm: test-cluster-linode
namespace: kube-system
options:
timeout: 5m
wait: true
waitForJobs: true
repoURL: https://linode.github.io/linode-cloud-controller-manager/
valuesTemplate: |-
secretRef:
name: "linode-token-region"
image:
pullPolicy: IfNotPresent
version: v0.4.4
---
apiVersion: addons.cluster.x-k8s.io/v1beta1
kind: ClusterResourceSet
metadata:
name: test-cluster-cilium-policy
namespace: default
spec:
clusterSelector:
matchLabels:
cluster: test-cluster
resources:
- kind: ConfigMap
name: test-cluster-cilium-policy
strategy: Reconcile
---
apiVersion: addons.cluster.x-k8s.io/v1beta1
kind: ClusterResourceSet
metadata:
name: test-cluster-crs-0
namespace: default
spec:
clusterSelector:
matchLabels:
crs: test-cluster-crs
resources:
- kind: Secret
name: linode-test-cluster-crs-0
strategy: ApplyOnce
---
apiVersion: addons.cluster.x-k8s.io/v1beta1
kind: ClusterResourceSet
metadata:
name: test-cluster-etcd-cilium-policy
namespace: default
spec:
clusterSelector:
matchLabels:
cluster: test-cluster
resources:
- kind: ConfigMap
name: test-cluster-etcd-cilium-policy
strategy: Reconcile
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
name: test-cluster-md-0
namespace: default
spec:
template:
spec:
format: ignition
ignition:
containerLinuxConfig:
additionalConfig: |
storage:
links:
- path: /etc/extensions/kubernetes.raw
hard: false
target: /opt/extensions/kubernetes/kubernetes-v1.29.5-x86-64.raw
files:
- path: /etc/sysupdate.kubernetes.d/kubernetes-v1.29.conf
mode: 0644
contents:
remote:
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-v1.29.conf
- path: /opt/extensions/kubernetes/kubernetes-v1.29.5-x86-64.raw
contents:
remote:
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-v1.29.5-x86-64.raw
systemd:
units:
- name: kubeadm.service
enabled: true
dropins:
- name: 10-flatcar.conf
contents: |
[Unit]
Requires=containerd.service coreos-metadata.service
After=containerd.service coreos-metadata.service
[Service]
EnvironmentFile=/run/metadata/flatcar
joinConfiguration:
nodeRegistration:
kubeletExtraArgs:
cloud-provider: external
provider-id: linode:///${COREOS_AKAMAI_INSTANCE_ID}
name: ${COREOS_AKAMAI_INSTANCE_LABEL}
preKubeadmCommands:
- export COREOS_AKAMAI_INSTANCE_LABEL=${COREOS_AKAMAI_INSTANCE_LABEL%.*}
- export COREOS_AKAMAI_INSTANCE_ID=${COREOS_AKAMAI_INSTANCE_ID}
- envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
- mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
labels:
ccm: test-cluster-linode
cluster: test-cluster
cni: test-cluster-cilium
crs: test-cluster-crs
csi: test-cluster-linode
vpcless: "true"
name: test-cluster
namespace: default
spec:
clusterNetwork:
pods:
cidrBlocks:
- 10.192.0.0/10
controlPlaneRef:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlane
name: test-cluster-control-plane
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeCluster
name: test-cluster
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineDeployment
metadata:
name: test-cluster-md-0
namespace: default
spec:
clusterName: test-cluster
replicas: 0
selector:
matchLabels: null
template:
spec:
bootstrap:
configRef:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
name: test-cluster-md-0
clusterName: test-cluster
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeMachineTemplate
name: test-cluster-md-0
version: v1.29.5
---
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlane
metadata:
name: test-cluster-control-plane
namespace: default
spec:
kubeadmConfigSpec:
clusterConfiguration:
apiServer:
extraArgs:
cloud-provider: external
controllerManager:
extraArgs:
cloud-provider: external
etcd:
local:
dataDir: /var/lib/etcd_data/etcd
extraArgs:
quota-backend-bytes: "8589934592"
format: ignition
ignition:
containerLinuxConfig:
additionalConfig: |
storage:
links:
- path: /etc/extensions/kubernetes.raw
hard: false
target: /opt/extensions/kubernetes/kubernetes-v1.29.5-x86-64.raw
files:
- path: /etc/sysupdate.kubernetes.d/kubernetes-v1.29.conf
mode: 0644
contents:
remote:
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-v1.29.conf
- path: /opt/extensions/kubernetes/kubernetes-v1.29.5-x86-64.raw
contents:
remote:
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-v1.29.5-x86-64.raw
systemd:
units:
- name: kubeadm.service
enabled: true
dropins:
- name: 10-flatcar.conf
contents: |
[Unit]
Requires=containerd.service coreos-metadata.service
After=containerd.service coreos-metadata.service
[Service]
EnvironmentFile=/run/metadata/flatcar
initConfiguration:
nodeRegistration:
kubeletExtraArgs:
cloud-provider: external
provider-id: linode:///${COREOS_AKAMAI_INSTANCE_UUID}
name: ${COREOS_AKAMAI_LABEL}
joinConfiguration:
nodeRegistration:
kubeletExtraArgs:
cloud-provider: external
provider-id: linode:///${COREOS_AKAMAI_INSTANCE_ID}
name: ${COREOS_AKAMAI_INSTANCE_LABEL}
preKubeadmCommands:
- export COREOS_AKAMAI_INSTANCE_LABEL=${COREOS_AKAMAI_INSTANCE_LABEL%.*}
- export COREOS_AKAMAI_INSTANCE_ID=${COREOS_AKAMAI_INSTANCE_ID}
- envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
- mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeMachineTemplate
name: test-cluster-control-plane
replicas: 1
version: v1.29.5
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeCluster
metadata:
name: test-cluster
namespace: default
spec:
credentialsRef:
name: test-cluster-credentials
region: us-ord
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeMachineTemplate
metadata:
name: test-cluster-control-plane
namespace: default
spec:
template:
spec:
image: private/25941695
interfaces:
- primary: true
purpose: public
region: us-ord
type: g6-standard-2
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: LinodeMachineTemplate
metadata:
name: test-cluster-md-0
namespace: default
spec:
template:
spec:
image: private/25941695
interfaces:
- primary: true
purpose: public
region: us-ord
type: g6-standard-2
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment