calico-issue

ip-10-111-38-8.ap-south-1.compute.internal

---
# the node doesn't exist
k get nodes | grep ip-10-111-38-8.ap-south-1.compute.internal
<blank>
calico-kube-controllers log:
022-10-26 17:06:56.985 [INFO][1] ipam.go 1174: Calico Node referenced in IPAM data does not exist error=resource does not exist: Node(<node-name>) with error: nodes "<node-name>" not found
...
...
│ 2022-10-26 17:09:13.843 [INFO][1] ipam_block_reader_writer.go 334: Block must be empty but is not empty, refusing to remove affinity. host="<node-name>" subnet="subnet/cidr"                         │
│ 2022-10-26 17:09:13.882 [ERROR][1] ipam.go 1101: Failed to release block affinities for node calicoNode="<node-anme>" error=block 'subnet/cidr' is not empty
---
# blockaffinity does exist
kubectl get blockaffinity | grepip-10-111-38-8.ap-south-1.compute.internal
ip-10-111-38-8.ap-south-1.compute.internal-192-168-24-64-26      31h

kubectl get blockaffinity ip-10-111-38-8.ap-south-1.compute.internal-192-168-24-64-26  -o yaml
apiVersion: crd.projectcalico.org/v1
kind: BlockAffinity
metadata:
  annotations:
    projectcalico.org/metadata: '{"creationTimestamp":null}'
  creationTimestamp: "2022-10-25T09:04:35Z"
  generation: 2
  name: ip-10-111-38-8.ap-south-1.compute.internal-192-168-24-64-26
spec:
  cidr: 192.168.24.64/26
  deleted: "false"
  node: ip-10-111-38-8.ap-south-1.compute.internal
  state: confirmed
---
# ipamblocks does exist
k get ipamblocks | grep 192-168-24-64-26
192-168-24-64-26    32h

k get ipamblocks 192-168-24-64-26 -o yaml
apiVersion: crd.projectcalico.org/v1
kind: IPAMBlock
metadata:
  annotations:
    projectcalico.org/metadata: '{"creationTimestamp":null}'
  creationTimestamp: "2022-10-25T09:04:35Z"
  generation: 626
  name: 192-168-24-64-26
spec:
  affinity: host:ip-10-111-38-8.ap-south-1.compute.internal
  allocations:
  - null
  - null
  - null
  - null
  - null
  - null
  - 16
  - null
  - null
  - null
  - null
  - null
  - null
  - null
  - null
  - null
  - null
  - null
  - null
  - null
  - null
  - null
  - 22
  - null
  - 19
  - 0
  - 1
  - 2
  - 3
  - 4
  - 18
  - 5
  - 6
  - null
  - null
  - 9
  - 7
  - 8
  - null
  - null
  - 10
  - null
  - 17
  - 15
  - null
  - null
  - null
  - 21
  - null
  - 20
  - null
  - null
  - null
  - 11
  - null
  - null
  - null
  - null
  - 12
  - 13
  - null
  - 14
  - null
  - null
  attributes:
  - handle_id: vxlan-tunnel-addr-ip-10-111-16-37.ap-south-1.compute.internal
    secondary:
      node: ip-10-111-16-37.ap-south-1.compute.internal
      type: vxlanTunnelAddress
  - handle_id: k8s-pod-network.1acc3b6bb01b28fdf45a3bd2f05d11f69d18450d92eeb87cffa78802dd0db5f0
    secondary:
      namespace: istio-system
      node: ip-10-111-16-37.ap-south-1.compute.internal
      pod: istio-cni-node-nnk78
      timestamp: 2022-10-25 16:18:50.545831941 +0000 UTC
  - handle_id: k8s-pod-network.fb6c43ede7cdab0e627529e7bc1030f516f87a3fe0888a194f42b6176731d826
    secondary:
      namespace: zone-logging
      node: ip-10-111-16-37.ap-south-1.compute.internal
      pod: olympus-zone-logging-zeta-shipper-gbk5w
      timestamp: 2022-10-25 16:18:50.59412689 +0000 UTC
  - handle_id: k8s-pod-network.0ac38916a38872a27796828647c21236117bfb48ee8c4b23405373ff25226c85
    secondary:
      namespace: zone-infra
      node: ip-10-111-16-37.ap-south-1.compute.internal
      pod: olympus-zone-infra-spot-termination-exporter-96bks
      timestamp: 2022-10-25 16:18:51.372508533 +0000 UTC
  - handle_id: k8s-pod-network.dac42b0dbe3b7b0aff3d6d1029148cbc88608afa4e0e4aca3544cf65f31b2378
    secondary:
      namespace: aura
      node: ip-10-111-16-37.ap-south-1.compute.internal
      pod: redis-cluster-server-2
      timestamp: 2022-10-25 16:29:50.058404441 +0000 UTC
  - handle_id: k8s-pod-network.794feeefa2bae45f34904d49ca328ed8d7ad0d6a1200dbf4fac7b6fb6c478184
    secondary:
      namespace: luminosrewards
      node: ip-10-111-16-37.ap-south-1.compute.internal
      pod: himalia-decision-engine-crud-85d59b7cc6-67vvs
      timestamp: 2022-10-25 16:33:39.849329372 +0000 UTC
  - handle_id: k8s-pod-network.8a0217fb57b1d63a3177c205af9dbadfe01ad4e53360a2ff02819f776a2ec4a1
    secondary:
      namespace: castle
      node: ip-10-111-16-37.ap-south-1.compute.internal
      pod: castle-heap-dump-pusher-s3-qndqk
      timestamp: 2022-10-25 16:36:05.351072321 +0000 UTC
  - handle_id: k8s-pod-network.42dca1c03aea01deec541d30930d33d61809744bd4dd0715b12793c56ff76dee
    secondary:
      namespace: zone-infra
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: olympus-zone-infra-spot-termination-exporter-v6zwj
      timestamp: 2022-10-26 07:51:22.53198364 +0000 UTC
  - handle_id: vxlan-tunnel-addr-ip-10-111-30-12.ap-south-1.compute.internal
    secondary:
      node: ip-10-111-30-12.ap-south-1.compute.internal
      type: vxlanTunnelAddress
  - handle_id: k8s-pod-network.ec2f78e2bff1176dfb307027f1c6c5f8bdf35732e0f5e0dd58fe1b5b8fbbfd67
    secondary:
      namespace: istio-system
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: istio-cni-node-8mqz5
      timestamp: 2022-10-26 07:51:23.734741707 +0000 UTC
  - handle_id: k8s-pod-network.c3933818adde3f0d83628360e3e49d950183678603f69c6f214b5ed633dadc4a
    secondary:
      namespace: zone-logging
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: olympus-zone-logging-zeta-shipper-hct6f
      timestamp: 2022-10-26 07:51:23.775085487 +0000 UTC
  - handle_id: k8s-pod-network.ea1c7667aa0c841056d28f6b77643e002e40ed79828c47dddaaea13e51ab05c5
    secondary:
      namespace: castle
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: castle-heap-dump-pusher-s3-94qm6
      timestamp: 2022-10-26 07:51:37.310854519 +0000 UTC
  - handle_id: k8s-pod-network.5b6f2261ed30fbfdaf5470071f039b4956c02c5b97e7d9d674712d9132733303
    secondary:
      namespace: aura
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: principalliquidationworker-flinkcluster-jobmanager-0
      timestamp: 2022-10-26 07:52:29.003594998 +0000 UTC
  - handle_id: k8s-pod-network.d31d3302703a5c267609413dfded5c360c3d204a14a4356a9cd9874338c4b261
    secondary:
      namespace: aura
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: emeraldaccrualworker-flinkcluster-jobmanager-0
      timestamp: 2022-10-26 07:52:30.120010975 +0000 UTC
  - handle_id: k8s-pod-network.f845450af5a5ad04e0a3283d39b7d5e86193672d5bdaf98c8ef4cd0ba2b8ed0f
    secondary:
      namespace: jupiter
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: locker-5bddc6ccc6-mhx6r
      timestamp: 2022-10-26 07:53:43.121975446 +0000 UTC
  - handle_id: k8s-pod-network.02abb40ab66f6384748b07c6f942fa010bf1e421edaf77d5b6132d7bda64f373
    secondary:
      namespace: watch
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: sparta-84f984dfc7-tzgb8
      timestamp: 2022-10-26 08:03:58.8936858 +0000 UTC
  - handle_id: k8s-pod-network.89e02dfcc61d197f4ddc52741bbfbe11911710a51cb9c90e999ac70685921944
    secondary:
      namespace: plutus-perf
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: redis-cluster-haproxy-6489d68df-mkwjm
      timestamp: 2022-10-26 08:12:09.079594891 +0000 UTC
  - handle_id: k8s-pod-network.2d73ab6c70adf658fc28031374d8236a332e761884abe47a3382ce4ccb73551f
    secondary:
      namespace: athena-pci
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: sparta-dashboard-89f9995bf-cvcbl
      timestamp: 2022-10-26 08:12:09.389434864 +0000 UTC
  - handle_id: k8s-pod-network.e90b34c39c9659aab83964b148405e5a535f3904aaec3718d8a8636a2402244d
    secondary:
      namespace: turbo
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: hdfcturboonboarding-ganymede-shared-postgrest-768954c8bc-sjhn8
      timestamp: 2022-10-26 08:12:10.305185964 +0000 UTC
  - handle_id: k8s-pod-network.3bf925fe750637f07ed51d97fb0820019d3dfafbe5fb705d23856d6c5bdafb37
    secondary:
      namespace: emerald
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: emerald-ganymede-shared-postgrest-7c67cfd586-rkxrl
      timestamp: 2022-10-26 08:12:11.573716327 +0000 UTC
  - handle_id: k8s-pod-network.f6778051df515c5b73ef9b728381d7ae1929b9ca9911949625cd91b5c0f9503d
    secondary:
      namespace: cipher
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: nginx-s3-proxy-658bb44658-hxgxk
      timestamp: 2022-10-26 08:12:12.907402535 +0000 UTC
  - handle_id: k8s-pod-network.bd559b0a17c5a3345bbf98633255ffe26e3dbc501337605a47b0fd0ec0fe02da
    secondary:
      namespace: aura
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: failuresimulatorworker-flinkcluster-jobmanager-0
      timestamp: 2022-10-26 08:12:30.610191586 +0000 UTC
  - handle_id: k8s-pod-network.9bb3f4fbb7c2b066a3667bf383d54d2c7c4ed7c877f751653ba0ada23dfe656b
    secondary:
      namespace: athena-pci
      node: ip-10-111-30-12.ap-south-1.compute.internal
      pod: redis-cluster-server-2
      timestamp: 2022-10-26 08:36:21.639501729 +0000 UTC
  cidr: 192.168.24.64/26
  deleted: false
  sequenceNumber: 0
  strictAffinity: false
  unallocated:
  - 1
  - 41
  - 38
  - 20
  - 11
  - 62
  - 44
  - 10
  - 9
  - 13
  - 34
  - 21
  - 17
  - 5
  - 19
  - 45
  - 56
  - 4
  - 51
  - 39
  - 3
  - 7
  - 55
  - 57
  - 15
  - 16
  - 0
  - 2
  - 23
  - 12
  - 46
  - 60
  - 33
  - 52
  - 48
  - 18
  - 50
  - 14
  - 63
  - 8
  - 54