Skip to content

Instantly share code, notes, and snippets.

@dav1x

dav1x/Affected system

Created January 24, 2018 16:48
Show Gist options
  • Save dav1x/4ec464419c83da923f6f757c86f8e851 to your computer and use it in GitHub Desktop.
Save dav1x/4ec464419c83da923f6f757c86f8e851 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
Affected system
[root@master-0 ~]# curl -kv 10.129.1.4:5000
* About to connect() to 10.129.1.4 port 5000 (#0)
* Trying 10.129.1.4...
* No route to host
* Failed connect to 10.129.1.4:5000; No route to host
* Closing connection 0
curl: (7) Failed connect to 10.129.1.4:5000; No route to host
[root@master-0 ~]# ovs-ofctl -O openflow13 dump-flows br0
OFPST_FLOW reply (OF1.3) (xid=0x2):
cookie=0x0, duration=385.438s, table=0, n_packets=0, n_bytes=0, priority=250,ip,in_port=2,nw_dst=224.0.0.0/4 actions=drop
cookie=0x0, duration=385.546s, table=0, n_packets=0, n_bytes=0, priority=200,arp,in_port=1,arp_spa=10.129.0.0/16,arp_tpa=10.129.4.0/24 actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:10
cookie=0x0, duration=385.536s, table=0, n_packets=0, n_bytes=0, priority=200,ip,in_port=1,nw_src=10.129.0.0/16 actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:10
cookie=0x0, duration=385.512s, table=0, n_packets=0, n_bytes=0, priority=200,ip,in_port=1,nw_dst=10.129.0.0/16 actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:10
cookie=0x0, duration=385.411s, table=0, n_packets=9, n_bytes=378, priority=200,arp,in_port=2,arp_spa=10.129.4.1,arp_tpa=10.129.0.0/16 actions=goto_table:30
cookie=0x0, duration=385.401s, table=0, n_packets=0, n_bytes=0, priority=200,ip,in_port=2 actions=goto_table:30
cookie=0x0, duration=385.470s, table=0, n_packets=0, n_bytes=0, priority=150,in_port=1 actions=drop
cookie=0x0, duration=385.392s, table=0, n_packets=8, n_bytes=648, priority=150,in_port=2 actions=drop
cookie=0x0, duration=385.384s, table=0, n_packets=0, n_bytes=0, priority=100,arp actions=goto_table:20
cookie=0x0, duration=385.375s, table=0, n_packets=0, n_bytes=0, priority=100,ip actions=goto_table:20
cookie=0x0, duration=385.362s, table=0, n_packets=0, n_bytes=0, priority=0 actions=drop
cookie=0x0, duration=385.012s, table=10, n_packets=0, n_bytes=0, priority=100,tun_src=10.19.114.245 actions=goto_table:30
cookie=0x0, duration=384.975s, table=10, n_packets=0, n_bytes=0, priority=100,tun_src=10.19.114.243 actions=goto_table:30
cookie=0x0, duration=384.931s, table=10, n_packets=0, n_bytes=0, priority=100,tun_src=10.19.114.244 actions=goto_table:30
cookie=0x0, duration=384.905s, table=10, n_packets=0, n_bytes=0, priority=100,tun_src=10.19.114.242 actions=goto_table:30
cookie=0x0, duration=385.353s, table=10, n_packets=0, n_bytes=0, priority=0 actions=drop
cookie=0x0, duration=385.344s, table=20, n_packets=0, n_bytes=0, priority=0 actions=drop
cookie=0x0, duration=385.335s, table=21, n_packets=0, n_bytes=0, priority=0 actions=goto_table:30
cookie=0x0, duration=385.331s, table=30, n_packets=0, n_bytes=0, priority=300,arp,arp_tpa=10.129.4.1 actions=output:2
cookie=0x0, duration=385.266s, table=30, n_packets=0, n_bytes=0, priority=300,ip,nw_dst=10.129.4.1 actions=output:2
cookie=0x0, duration=385.325s, table=30, n_packets=0, n_bytes=0, priority=200,arp,arp_tpa=10.129.4.0/24 actions=goto_table:40
cookie=0x0, duration=385.199s, table=30, n_packets=0, n_bytes=0, priority=200,ip,nw_dst=10.129.4.0/24 actions=goto_table:70
cookie=0x0, duration=385.289s, table=30, n_packets=9, n_bytes=378, priority=100,arp,arp_tpa=10.129.0.0/16 actions=goto_table:50
cookie=0x0, duration=385.233s, table=30, n_packets=0, n_bytes=0, priority=100,ip,nw_dst=10.128.0.0/16 actions=goto_table:60
cookie=0x0, duration=385.182s, table=30, n_packets=0, n_bytes=0, priority=100,ip,nw_dst=10.129.0.0/16 actions=goto_table:90
cookie=0x0, duration=385.157s, table=30, n_packets=0, n_bytes=0, priority=50,ip,in_port=1,nw_dst=224.0.0.0/4 actions=goto_table:120
cookie=0x0, duration=385.129s, table=30, n_packets=0, n_bytes=0, priority=25,ip,nw_dst=224.0.0.0/4 actions=goto_table:110
cookie=0x0, duration=385.111s, table=30, n_packets=0, n_bytes=0, priority=0,ip actions=goto_table:100
cookie=0x0, duration=385.105s, table=30, n_packets=0, n_bytes=0, priority=0,arp actions=drop
cookie=0x0, duration=385.100s, table=40, n_packets=0, n_bytes=0, priority=0 actions=drop
cookie=0x0, duration=385.003s, table=50, n_packets=9, n_bytes=378, priority=100,arp,arp_tpa=10.129.1.0/24 actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:10.19.114.245->tun_dst,output:1
cookie=0x0, duration=384.960s, table=50, n_packets=0, n_bytes=0, priority=100,arp,arp_tpa=10.129.2.0/24 actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:10.19.114.243->tun_dst,output:1
cookie=0x0, duration=384.922s, table=50, n_packets=0, n_bytes=0, priority=100,arp,arp_tpa=10.129.3.0/24 actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:10.19.114.244->tun_dst,output:1
cookie=0x0, duration=384.895s, table=50, n_packets=0, n_bytes=0, priority=100,arp,arp_tpa=10.129.0.0/24 actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:10.19.114.242->tun_dst,output:1
cookie=0x0, duration=385.094s, table=50, n_packets=0, n_bytes=0, priority=0 actions=drop
cookie=0x0, duration=385.090s, table=60, n_packets=0, n_bytes=0, priority=200,reg0=0 actions=output:2
cookie=0x0, duration=384.758s, table=60, n_packets=0, n_bytes=0, priority=100,ip,nw_dst=10.128.0.1,nw_frag=later actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=384.727s, table=60, n_packets=0, n_bytes=0, priority=100,ip,nw_dst=10.128.96.180,nw_frag=later actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=384.692s, table=60, n_packets=0, n_bytes=0, priority=100,ip,nw_dst=10.128.235.33,nw_frag=later actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=384.681s, table=60, n_packets=0, n_bytes=0, priority=100,ip,nw_dst=10.128.14.21,nw_frag=later actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=384.749s, table=60, n_packets=0, n_bytes=0, priority=100,tcp,nw_dst=10.128.0.1,tp_dst=443 actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=384.742s, table=60, n_packets=0, n_bytes=0, priority=100,udp,nw_dst=10.128.0.1,tp_dst=53 actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=384.735s, table=60, n_packets=0, n_bytes=0, priority=100,tcp,nw_dst=10.128.0.1,tp_dst=53 actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=384.719s, table=60, n_packets=0, n_bytes=0, priority=100,tcp,nw_dst=10.128.96.180,tp_dst=80 actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=384.714s, table=60, n_packets=0, n_bytes=0, priority=100,tcp,nw_dst=10.128.96.180,tp_dst=443 actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=384.704s, table=60, n_packets=0, n_bytes=0, priority=100,tcp,nw_dst=10.128.96.180,tp_dst=1936 actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=384.686s, table=60, n_packets=0, n_bytes=0, priority=100,tcp,nw_dst=10.128.235.33,tp_dst=5000 actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=384.668s, table=60, n_packets=0, n_bytes=0, priority=100,tcp,nw_dst=10.128.14.21,tp_dst=9000 actions=load:0->NXM_NX_REG1[],load:0x2->NXM_NX_REG2[],goto_table:80
cookie=0x0, duration=385.084s, table=60, n_packets=0, n_bytes=0, priority=0 actions=drop
cookie=0x0, duration=385.075s, table=70, n_packets=0, n_bytes=0, priority=0 actions=drop
cookie=0x0, duration=385.069s, table=80, n_packets=0, n_bytes=0, priority=300,ip,nw_src=10.129.4.1 actions=output:NXM_NX_REG2[]
cookie=0x0, duration=385.002s, table=80, n_packets=0, n_bytes=0, priority=200,reg0=0 actions=output:NXM_NX_REG2[]
cookie=0x0, duration=384.991s, table=80, n_packets=0, n_bytes=0, priority=200,reg1=0 actions=output:NXM_NX_REG2[]
cookie=0x0, duration=385.066s, table=80, n_packets=0, n_bytes=0, priority=0 actions=drop
cookie=0x0, duration=384.992s, table=90, n_packets=0, n_bytes=0, priority=100,ip,nw_dst=10.129.1.0/24 actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:10.19.114.245->tun_dst,output:1
cookie=0x0, duration=384.952s, table=90, n_packets=0, n_bytes=0, priority=100,ip,nw_dst=10.129.2.0/24 actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:10.19.114.243->tun_dst,output:1
cookie=0x0, duration=384.917s, table=90, n_packets=0, n_bytes=0, priority=100,ip,nw_dst=10.129.3.0/24 actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:10.19.114.244->tun_dst,output:1
cookie=0x0, duration=384.885s, table=90, n_packets=0, n_bytes=0, priority=100,ip,nw_dst=10.129.0.0/24 actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:10.19.114.242->tun_dst,output:1
cookie=0x0, duration=385.059s, table=90, n_packets=0, n_bytes=0, priority=0 actions=drop
cookie=0x0, duration=385.055s, table=100, n_packets=0, n_bytes=0, priority=0 actions=goto_table:101
cookie=0x0, duration=385.051s, table=101, n_packets=0, n_bytes=0, priority=51,tcp,nw_dst=10.19.114.241,tp_dst=53 actions=output:2
cookie=0x0, duration=385.048s, table=101, n_packets=0, n_bytes=0, priority=51,udp,nw_dst=10.19.114.241,tp_dst=53 actions=output:2
cookie=0x0, duration=385.042s, table=101, n_packets=0, n_bytes=0, priority=0 actions=output:2
cookie=0x0, duration=385.037s, table=110, n_packets=0, n_bytes=0, priority=0 actions=drop
cookie=0x0, duration=384.878s, table=111, n_packets=0, n_bytes=0, priority=100 actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:10.19.114.242->tun_dst,output:1,set_field:10.19.114.243->tun_dst,output:1,set_field:10.19.114.244->tun_dst,output:1,set_field:10.19.114.245->tun_dst,output:1,goto_table:120
cookie=0x0, duration=385.030s, table=120, n_packets=0, n_bytes=0, priority=0 actions=drop
cookie=0x0, duration=385.027s, table=253, n_packets=0, n_bytes=0, actions=note:01.05.00.00.00.00
[root@master-0 ~]# oc get hostsubnets
NAME HOST HOST IP SUBNET EGRESS IPS
app-0 app-0 10.19.114.244 10.129.3.0/24 []
infra-0 infra-0 10.19.114.245 10.129.1.0/24 []
master-0 master-0 10.19.114.241 10.129.5.0/24 []
master-1 master-1 10.19.114.242 10.129.0.0/24 []
master-2 master-2 10.19.114.243 10.129.2.0/24 []
[root@master-0 ~]# oc get nodes -o wide
NAME STATUS AGE VERSION EXTERNAL-IP OS-IMAGE KERNEL-VERSION
app-0 Ready 50m v1.7.6+a08f5eeb62 <none> Unknown 3.10.0-693.el7.x86_64
infra-0 Ready 50m v1.7.6+a08f5eeb62 <none> Unknown 3.10.0-693.el7.x86_64
master-0 Ready 7m v1.7.6+a08f5eeb62 <none> Unknown 3.10.0-693.el7.x86_64
master-1 Ready 22m v1.7.6+a08f5eeb62 10.19.114.242 Unknown 3.10.0-693.el7.x86_64
master-2 Ready 22m v1.7.6+a08f5eeb62 10.19.114.243 Unknown 3.10.0-693.el7.x86_64
[root@master-0 ~]# oc get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE
docker-registry-1-hwzdz 1/1 Running 0 47m 10.129.1.4 infra-0
registry-console-1-pl8dp 1/1 Running 0 45m 10.129.3.3 app-0
router-1-hmj62 1/1 Running 0 48m 10.19.114.245 infra-0
[root@master-0 ~]# curl -kv 10.129.1.4:5000
* About to connect() to 10.129.1.4 port 5000 (#0)
* Trying 10.129.1.4...
* No route to host
* Failed connect to 10.129.1.4:5000; No route to host
* Closing connection 0
curl: (7) Failed connect to 10.129.1.4:5000; No route to host
[root@master-0 ~]# oc get svc
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
docker-registry 10.128.235.33 <none> 5000/TCP 49m
kubernetes 10.128.0.1 <none> 443/TCP,53/UDP,53/TCP 59m
registry-console 10.128.14.21 <none> 9000/TCP 48m
router 10.128.96.180 <none> 80/TCP,443/TCP,1936/TCP 50m
[root@master-0 ~]# curl -kv 10.128.235.33:5000
* About to connect() to 10.128.235.33 port 5000 (#0)
* Trying 10.128.235.33...
* No route to host
* Failed connect to 10.128.235.33:5000; No route to host
* Closing connection 0
curl: (7) Failed connect to 10.128.235.33:5000; No route to host
[root@master-0 ~]# ip route
default via 10.19.115.254 dev ens192 proto static metric 100
10.19.114.0/23 dev ens192 proto kernel scope link src 10.19.114.241 metric 100
10.128.0.0/16 dev tun0
10.129.0.0/16 dev tun0 scope link
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1
[root@master-0 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens192: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
link/ether 00:50:56:a5:1e:97 brd ff:ff:ff:ff:ff:ff
inet 10.19.114.241/23 brd 10.19.115.255 scope global ens192
valid_lft forever preferred_lft forever
inet6 2620:52:0:1372:250:56ff:fea5:1e97/64 scope global mngtmpaddr dynamic
valid_lft 2591776sec preferred_lft 604576sec
inet6 fe80::250:56ff:fea5:1e97/64 scope link
valid_lft forever preferred_lft forever
3: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
link/ether b6:b7:5a:b2:39:7c brd ff:ff:ff:ff:ff:ff
6: br0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
link/ether 12:e8:8f:9b:9b:41 brd ff:ff:ff:ff:ff:ff
7: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN
link/ether 02:42:13:c3:12:de brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 scope global docker0
valid_lft forever preferred_lft forever
8: vxlan_sys_4789: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 65470 qdisc noqueue master ovs-system state UNKNOWN qlen 1000
link/ether ae:f7:78:50:65:ce brd ff:ff:ff:ff:ff:ff
inet6 fe80::acf7:78ff:fe50:65ce/64 scope link
valid_lft forever preferred_lft forever
9: tun0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN qlen 1000
link/ether 4e:4d:6e:1b:4d:a5 brd ff:ff:ff:ff:ff:ff
inet 10.129.4.1/24 brd 10.129.4.255 scope global tun0
valid_lft forever preferred_lft forever
inet6 fe80::4c4d:6eff:fe1b:4da5/64 scope link
valid_lft forever preferred_lft forever
master-0 is the affected node. After the installation I added the vsphere config file and only ran the following
commands before the issue was prevalent.
systemctl stop atomic-openshift-node
systemctl stop docker
systemctl restart iptables
systemctl start docker
systemctl start atomic-openshift-node
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment