Skip to content

Instantly share code, notes, and snippets.

@networkop
Last active July 26, 2021 21:12
Show Gist options
  • Save networkop/10efe3a8243ed9fb45c2e158d5319317 to your computer and use it in GitHub Desktop.
Save networkop/10efe3a8243ed9fb45c2e158d5319317 to your computer and use it in GitHub Desktop.
cilium-logs
level=error msg="Command execution failed" cmd="[tc filter replace dev lxc124b240e531e ingress prio 1 handle 1 bpf da obj 1743_next/bpf_lxc.o sec from-container]" error="exit status 1" subsys=datapath-loader
level=warning subsys=datapath-loader
level=warning msg="Prog section 'from-container' rejected: Argument list too long (7)!" subsys=datapath-loader
level=warning msg=" - Type: 3" subsys=datapath-loader
level=warning msg=" - Attach Type: 0" subsys=datapath-loader
level=warning msg=" - Instructions: 3609 (0 over limit)" subsys=datapath-loader
level=warning msg=" - License: GPL" subsys=datapath-loader
level=warning subsys=datapath-loader
level=warning msg="Verifier analysis:" subsys=datapath-loader
level=warning subsys=datapath-loader
level=warning msg="Skipped 154997 bytes, use 'verb' option for the full verbose log." subsys=datapath-loader
level=warning msg="[...]" subsys=datapath-loader
level=warning msg="???mmmm fp-184=mmmmmmmm fp-192=inv fp-200=inv fp-208=inv fp-216=inv16 fp-224=00000000 fp-232=mmmmmmmm fp-240=inv fp-248=inv fp-256=inv1 fp-264=inv fp-272=inv fp-280=00000000 fp-288=00000000 fp-296=00000000" subsys=datapath-loader
level=warning msg="1276: (61) r1 = *(u32 *)(r6 +0)" subsys=datapath-loader
level=warning msg="1277: (db) lock *(u64 *)(r0 +16) += r1" subsys=datapath-loader
level=warning msg=" R0_w=map_value(id=0,off=0,ks=8,vs=24,imm=0) R1_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff)) R8=inv1 R9=inv(id=64190,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0 fp-8=????0000 fp-32=mmmmmmmm fp-40=mmmmmmmm fp-80=mmmmmmmm fp-88=mmmmmmmm fp-96=mmmmmmmm fp-104=????0000 fp-112=0000mmmm fp-120=??mmmmmm fp-128=mmmmmmmm fp-136=??mmmmmm fp-144=????00mm fp-152=00000000 fp-160=0000mmmm fp-176=????mmmm fp-184=mmmmmmmm fp-192=inv fp-200=inv fp-208=inv fp-216=inv16 fp-224=00000000 fp-232=mmmmmmmm fp-240=inv fp-248=inv fp-256=inv1 fp-264=inv fp-272=inv fp-280=00000000 fp-288=00000000 fp-296=00000000" subsys=datapath-loader
level=warning msg=" R0_w=map_value(id=0,off=0,ks=8,vs=24,imm=0) R1_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff)) R8=inv1 R9=inv(id=64190,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0 fp-8=????0000 fp-32=mmmmmmmm fp-40=mmmmmmmm fp-80=mmmmmmmm fp-88=mmmmmmmm fp-96=mmmmmmmm fp-104=????0000 fp-112=0000mmmm fp-120=??mmmmmm fp-128=mmmmmmmm fp-136=??mmmmmm fp-144=????00mm fp-152=00000000 fp-160=0000mmmm fp-176=????mmmm fp-184=mmmmmmmm fp-192=inv fp-200=inv fp-208=inv fp-216=inv16 fp-224=00000000 fp-232=mmmmmmmm fp-240=inv fp-248=inv fp-256=inv1 fp-264=inv fp-272=inv fp-280=00000000 fp-288=00000000 fp-296=00000000" subsys=datapath-loader
level=warning msg="1278: (18) r2 = 0xffffff4b" subsys=datapath-loader
level=warning msg="1280: (b7) r1 = 2" subsys=datapath-loader
level=warning msg="1281: (71) r3 = *(u8 *)(r0 +2)" subsys=datapath-loader
level=warning msg=" R0_w=map_value(id=0,off=0,ks=8,vs=24,imm=0) R1_w=inv2 R2_w=inv4294967115 R6=ctx(id=0,off=0,imm=0) R7=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff)) R8=inv1 R9=inv(id=64190,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0 fp-8=????0000 fp-32=mmmmmmmm fp-40=mmmmmmmm fp-80=mmmmmmmm fp-88=mmmmmmmm fp-96=mmmmmmmm fp-104=????0000 fp-112=0000mmmm fp-120=??mmmmmm fp-128=mmmmmmmm fp-136=??mmmmmm fp-144=????00mm fp-152=00000000 fp-160=0000mmmm fp-176=????mmmm fp-184=mmmmmmmm fp-192=inv fp-200=inv fp-208=inv fp-216=inv16 fp-224=00000000 fp-232=mmmmmmmm fp-240=inv fp-248=inv fp-256=inv1 fp-264=inv fp-272=inv fp-280=00000000 fp-288=00000000 fp-296=00000000" subsys=datapath-loader
level=warning msg="1282: (57) r3 &= 1" subsys=datapath-loader
level=warning msg="1283: (55) if r3 != 0x0 goto pc+2294" subsys=datapath-loader
level=warning msg=" R0=map_value(id=0,off=0,ks=8,vs=24,imm=0) R1=inv2 R2=inv4294967115 R3=inv0 R6=ctx(id=0,off=0,imm=0) R7=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff)) R8=inv1 R9=inv(id=64190,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0 fp-8=????0000 fp-32=mmmmmmmm fp-40=mmmmmmmm fp-80=mmmmmmmm fp-88=mmmmmmmm fp-96=mmmmmmmm fp-104=????0000 fp-112=0000mmmm fp-120=??mmmmmm fp-128=mmmmmmmm fp-136=??mmmmmm fp-144=????00mm fp-152=00000000 fp-160=0000mmmm fp-176=????mmmm fp-184=mmmmmmmm fp-192=inv fp-200=inv fp-208=inv fp-216=inv16 fp-224=00000000 fp-232=mmmmmmmm fp-240=inv fp-248=inv fp-256=inv1 fp-264=inv fp-272=inv fp-280=00000000 fp-288=00000000 fp-296=00000000" subsys=datapath-loader
level=warning msg="1284: (b7) r1 = 2" subsys=datapath-loader
level=warning msg="1285: (69) r0 = *(u16 *)(r0 +0)" subsys=datapath-loader
level=warning msg=" R0_w=map_value(id=0,off=0,ks=8,vs=24,imm=0) R1_w=inv2 R2=inv4294967115 R3=inv0 R6=ctx(id=0,off=0,imm=0) R7=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff)) R8=inv1 R9=inv(id=64190,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0 fp-8=????0000 fp-32=mmmmmmmm fp-40=mmmmmmmm fp-80=mmmmmmmm fp-88=mmmmmmmm fp-96=mmmmmmmm fp-104=????0000 fp-112=0000mmmm fp-120=??mmmmmm fp-128=mmmmmmmm fp-136=??mmmmmm fp-144=????00mm fp-152=00000000 fp-160=0000mmmm fp-176=????mmmm fp-184=mmmmmmmm fp-192=inv fp-200=inv fp-208=inv fp-216=inv16 fp-224=00000000 fp-232=mmmmmmmm fp-240=inv fp-248=inv fp-256=inv1 fp-264=inv fp-272=inv fp-280=00000000 fp-288=00000000 fp-296=00000000" subsys=datapath-loader
level=warning msg="1286: (57) r0 &= 65535" subsys=datapath-loader
level=warning msg="1287: (18) r8 = 0xffffff5d" subsys=datapath-loader
level=warning msg="1289: (79) r2 = *(u64 *)(r10 -256)" subsys=datapath-loader
level=warning msg="BPF program is too large. Processed 1000001 insn" subsys=datapath-loader
level=warning msg="processed 1000001 insns (limit 1000000) max_states_per_insn 26 total_states 64937 peak_states 2278 mark_read 93" subsys=datapath-loader
level=warning subsys=datapath-loader
level=warning msg="JIT disabled, but 1/5 tail call maps in the program have JITed owner!" subsys=datapath-loader
level=warning msg="Error fetching program/map!" subsys=datapath-loader
level=warning msg="Unable to load program" subsys=datapath-loader
level=warning msg="JoinEP: Failed to load program" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 error="Failed to load tc filter: exit status 1" file-path=1743_next/bpf_lxc.o identity=64492 ipv4= ipv6= k8sPodName=/ subsys=datapath-loader veth=lxc124b240e531e
level=error msg="Error while rewriting endpoint BPF program" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 error="Failed to load tc filter: exit status 1" identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Reverting endpoint changes after BPF regeneration failed" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Reverting proxy redirect removals" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Finished reverting proxy redirect removals" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Reverting proxy redirect additions" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Finished reverting proxy redirect additions" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Finished reverting endpoint changes after BPF regeneration failed" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=warning msg="generating BPF for endpoint failed, keeping stale directory." containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 file-path=1743_next_fail identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="removing directory" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 directory=1743_next_fail endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="removing directory" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 directory=1743_next endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Completed endpoint regeneration with no pending regeneration requests" code=OK containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 endpointState=ready identity=64492 ipv4= ipv6= k8sPodName=/ policyRevision=0 subsys=endpoint type=0
level=warning msg="Regeneration of endpoint failed" bpfCompilation=8.6360454s bpfLoadProg=1m53.4612753s bpfWaitForELF=8.636554s bpfWriteELF="611.2µs" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 error="Failed to load tc filter: exit status 1" identity=64492 ipv4= ipv6= k8sPodName=/ mapSync="56.2µs" policyCalculation="160.8µs" prepareBuild=1.4304ms proxyConfiguration="11.8µs" proxyPolicyCalculation=300ns proxyWaitForAck=0s reason="updated security labels" subsys=endpoint total=2m2.1186116s waitingForCTClean=1.5698ms waitingForLock="139.3µs"
level=debug msg="Error regenerating endpoint: Failed to load tc filter: exit status 1" code=Failure containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 endpointState=ready identity=64492 ipv4= ipv6= k8sPodName=/ policyRevision=0 subsys=endpoint type=200
level=debug msg="EventQueue event processing statistics" eventConsumeOffQueueWaitTime="130.6µs" eventEnqueueWaitTime="1.7µs" eventHandlingDuration=2m2.1190834s eventType="*endpoint.EndpointRegenerationEvent" name=endpoint-1743 subsys=eventqueue
level=error msg="endpoint regeneration failed" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 error="Failed to load tc filter: exit status 1" identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="received signal that regeneration failed" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Triggering endpoint regeneration due to retrying regeneration" code=OK containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 endpointState=waiting-to-regenerate identity=64492 ipv4= ipv6= k8sPodName=/ policyRevision=0 subsys=endpoint type=0
level=debug msg="regeneration was cancelled" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Controller func execution time: 2m2.1219818s" name=endpoint-1743-regeneration-recovery subsys=controller uuid=dac899b9-ee53-11eb-8127-5a6c0bcf121c
level=debug msg="Controller run failed" consecutiveErrors=1 error="regeneration recovery failed" name=endpoint-1743-regeneration-recovery subsys=controller uuid=dac899b9-ee53-11eb-8127-5a6c0bcf121c
level=debug msg="Deleting endpoint" code=OK containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 endpointState=disconnecting identity=64492 ipv4= ipv6= k8sPodName=/ policyRevision=0 subsys=endpoint type=0
level=debug msg="Registered BPF map" path=/sys/fs/bpf/tc/globals/cilium_call_policy subsys=bpf
level=debug msg="Unregistered BPF map" path=/sys/fs/bpf/tc/globals/cilium_call_policy subsys=bpf
level=debug msg="Released IP" ip=10.0.2.10 owner=default/net-tshoot-dkgbn subsys=ipam
level=debug msg="Unregistered BPF map" path=/sys/fs/bpf/tc/globals/cilium_policy_01743 subsys=bpf
level=debug msg="Removing identity from the identity manager" identity=64492 subsys=identitymanager
level=info msg="Releasing key" key="[k8s:io.cilium.k8s.namespace.labels.kubernetes.io/metadata.name=default k8s:io.cilium.k8s.policy.cluster=default k8s:io.cilium.k8s.policy.serviceaccount=default k8s:io.kubernetes.pod.namespace=default k8s:name=net-tshoot]" subsys=allocator
level=debug msg="removing directory" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 directory=1743 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="removing directory" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 directory=1743_next_fail endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="removing directory" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 directory=1743_next endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="removing directory" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 directory=1743_stale endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Removed controller" name=resolve-identity-1743 subsys=controller uuid=dac8dedc-ee53-11eb-8127-5a6c0bcf121c
level=debug msg="Removed controller" name=endpoint-1743-regeneration-recovery subsys=controller uuid=dac899b9-ee53-11eb-8127-5a6c0bcf121c
level=debug msg="Removed controller" name="sync-to-k8s-ciliumendpoint (1743)" subsys=controller uuid=dac89d03-ee53-11eb-8127-5a6c0bcf121c
level=debug msg="Registered BPF map" path=/sys/fs/bpf/tc/globals/cilium_ct4_global subsys=bpf
level=debug msg="Registered BPF map" path=/sys/fs/bpf/tc/globals/cilium_snat_v4_external subsys=bpf
level=debug msg="Unregistered BPF map" path=/sys/fs/bpf/tc/globals/cilium_snat_v4_external subsys=bpf
level=debug msg="Registered BPF map" path=/sys/fs/bpf/tc/globals/cilium_ct_any4_global subsys=bpf
level=debug msg="Registered BPF map" path=/sys/fs/bpf/tc/globals/cilium_snat_v4_external subsys=bpf
level=debug msg="Unregistered BPF map" path=/sys/fs/bpf/tc/globals/cilium_snat_v4_external subsys=bpf
level=debug msg="Unregistered BPF map" path=/sys/fs/bpf/tc/globals/cilium_ct_any4_global subsys=bpf
level=debug msg="Unregistered BPF map" path=/sys/fs/bpf/tc/globals/cilium_ct4_global subsys=bpf
level=debug msg="Endpoint removed" code=OK containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 endpointState=disconnected identity=64492 ipv4= ipv6= k8sPodName=/ policyRevision=0 subsys=endpoint type=0
level=info msg="Removed endpoint" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Waiting for proxy updates to complete..." containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=debug msg="Wait time for proxy updates: 19.6µs" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 identity=64492 ipv4= ipv6= k8sPodName=/ subsys=endpoint
level=warning msg="Ignoring error while deleting endpoint" endpointID=1743 error="Unable to delete key 10.0.2.10:0 from /sys/fs/bpf/tc/globals/cilium_lxc: unable to delete element 10.0.2.10:0 from map cilium_lxc: no such file or directory" subsys=daemon
level=warning msg="Ignoring error while deleting endpoint" endpointID=1743 error="unable to remove endpoint from global policy map: unable to delete element Endpoint: 1743 from map cilium_call_policy: no such file or directory" subsys=daemon
level=info msg="API call has been processed" name=endpoint-delete processingDuration=31.5965007s subsys=rate totalDuration=31.5966083s uuid=10be79e7-ee54-11eb-8127-5a6c0bcf121c waitDurationTotal="82.8µs"
level=warning msg="Creation of endpoint failed" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=1743 error="timeout while waiting for initial endpoint generation to complete" identity=64492 ipv4= ipv6= k8sPodName=/ subsys=daemon
level=info msg="API call has been processed" error="timeout while waiting for initial endpoint generation to complete" name=endpoint-create processingDuration=2m2.1280066s subsys=rate totalDuration=2m2.128069s uuid=dac8877f-ee53-11eb-8127-5a6c0bcf121c waitDurationTotal=0s
level=debug msg="Removing identity references from policy cache" identity=64492 subsys=policy
level=debug msg="Finished cleaning policy cache" identity=64492 subsys=policy
level=debug msg="Shutting down controller" name=resolve-identity-1743 subsys=controller uuid=dac8dedc-ee53-11eb-8127-5a6c0bcf121c
level=debug msg="Shutting down controller" name=endpoint-1743-regeneration-recovery subsys=controller uuid=dac899b9-ee53-11eb-8127-5a6c0bcf121c
level=debug msg="Shutting down controller" name="sync-to-k8s-ciliumendpoint (1743)" subsys=controller uuid=dac89d03-ee53-11eb-8127-5a6c0bcf121c
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment