Skip to content

Instantly share code, notes, and snippets.

@rbo

rbo/NetworkPolicy-Problem.md

Last active November 17, 2019 20:51
Show Gist options
  • Save rbo/057b92f479a597327925d5454067f0d3 to your computer and use it in GitHub Desktop.
Save rbo/057b92f479a597327925d5454067f0d3 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
NetworkPolicy-Problem.md

My notes to "Bug 1768608 - NetworkPolicy not applied to router"

Enviroment

Baremetal OpenShift 4.2.0

$ oc version
Client Version: openshift-clients-4.3.0-201909231341
Server Version: 4.2.0
Kubernetes Version: v1.14.6+2e5ed54

Network policy documentation: https://docs.openshift.com/container-platform/4.2/networking/configuring-networkpolicy.html#nw-networkpolicy-multitenant-isolation_configuring-networkpolicy-plugin

Deploy demo application

oc new-project np-test

oc new-app nginx-example
oc logs -f bc/nginx-example

1) Check connection via router pods - PASS

$ oc get pods -n np-test -o wide -n np-test
NAME                    READY   STATUS    RESTARTS   AGE    IP            NODE        NOMINATED NODE   READINESS GATES
nginx-example-1-ls67j   1/1     Running   0          5d1h   10.131.0.14   compute-0   <none>           <none>

$ oc get pods -o wide -n openshift-ingress
NAME                             READY   STATUS    RESTARTS   AGE    IP              NODE        NOMINATED NODE   READINESS GATES
router-default-c89f748f7-qldrg   1/1     Running   0          5d5h   192.168.51.15   compute-2   <none>           <none>
router-default-c89f748f7-qxf7q   1/1     Running   0          5d5h   192.168.51.13   compute-0   <none>           <none>

$ oc rsh router-default-c89f748f7-qxf7q curl -I http://10.131.0.14:8080/
HTTP/1.1 200 OK
Server: nginx/1.12.1
Date: Sat, 16 Nov 2019 17:48:42 GMT
Content-Type: text/html
Content-Length: 37451
Last-Modified: Mon, 11 Nov 2019 16:14:50 GMT
Connection: keep-alive
ETag: "5dc988fa-924b"
Accept-Ranges: bytes

$ oc rsh router-default-c89f748f7-qldrg curl -I http://10.131.0.14:8080/
HTTP/1.1 200 OK
Server: nginx/1.12.1
Date: Sat, 16 Nov 2019 17:49:03 GMT
Content-Type: text/html
Content-Length: 37451
Last-Modified: Mon, 11 Nov 2019 16:14:50 GMT
Connection: keep-alive
ETag: "5dc988fa-924b"
Accept-Ranges: bytes

Result - PASS

Source Target Expected result Result
router pod on compute-0 app pod on compute-0 PASS PASS
router pod on compute-2 app pod on compute-0 PASS PASS

2) Create deny-all NetworkPolicy & check connection - FAIL

oc create -n np-test -f - <<EOF
kind: NetworkPolicy
apiVersion: networking.k8s.io/v1
metadata:
    name: deny-by-default
spec:
    podSelector:
    ingress: []
EOF

$ oc get networkpolicies -n np-test
NAME                           POD-SELECTOR   AGE
allow-from-openshift-ingress   <none>         7m28s
deny-by-default                <none>         4m20s

$ oc get pods -n np-test -o wide -n np-test
NAME                    READY   STATUS    RESTARTS   AGE    IP            NODE        NOMINATED NODE   READINESS GATES
nginx-example-1-ls67j   1/1     Running   0          5d1h   10.131.0.14   compute-0   <none>           <none>

$ oc get pods -o wide -n openshift-ingress
NAME                             READY   STATUS    RESTARTS   AGE    IP              NODE        NOMINATED NODE   READINESS GATES
router-default-c89f748f7-qldrg   1/1     Running   0          5d5h   192.168.51.15   compute-2   <none>           <none>
router-default-c89f748f7-qxf7q   1/1     Running   0          5d5h   192.168.51.13   compute-0   <none>           <none>

$ oc rsh router-default-c89f748f7-qxf7q curl -I http://10.131.0.14:8080/
HTTP/1.1 200 OK
Server: nginx/1.12.1
Date: Sat, 16 Nov 2019 17:54:59 GMT
Content-Type: text/html
Content-Length: 37451
Last-Modified: Mon, 11 Nov 2019 16:14:50 GMT
Connection: keep-alive
ETag: "5dc988fa-924b"
Accept-Ranges: bytes

$ oc rsh router-default-c89f748f7-qldrg curl -I http://10.131.0.14:8080/
$

Result - FAIL

Source Target Expected result Result
router pod on compute-0 app pod on compute-0 FAIL PASS
router pod on compute-2 app pod on compute-0 FAIL FAIL

Create allow-from-openshift-ingress NetworkPolicy & check connection - FAIL

oc create -n np-test -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-from-openshift-ingress
spec:
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          network.openshift.io/policy-group: ingress
  podSelector: {}
  policyTypes:
  - Ingress
EOF


$ oc get pods -n np-test -o wide -n np-test
NAME                    READY   STATUS    RESTARTS   AGE    IP            NODE        NOMINATED NODE   READINESS GATES
nginx-example-1-ls67j   1/1     Running   0          5d1h   10.131.0.14   compute-0   <none>           <none>
$ oc get pods -o wide -n openshift-ingress
NAME                             READY   STATUS    RESTARTS   AGE    IP              NODE        NOMINATED NODE   READINESS GATES
router-default-c89f748f7-qldrg   1/1     Running   0          5d5h   192.168.51.15   compute-2   <none>           <none>
router-default-c89f748f7-qxf7q   1/1     Running   0          5d5h   192.168.51.13   compute-0   <none>           <none>

$ oc rsh router-default-c89f748f7-qxf7q curl -I http://10.131.0.14:8080/
HTTP/1.1 200 OK
Server: nginx/1.12.1
Date: Sat, 16 Nov 2019 17:57:40 GMT
Content-Type: text/html
Content-Length: 37451
Last-Modified: Mon, 11 Nov 2019 16:14:50 GMT
Connection: keep-alive
ETag: "5dc988fa-924b"
Accept-Ranges: bytes

$ oc rsh router-default-c89f748f7-qldrg curl -I http://10.131.0.14:8080/
$

Result - FAIL

Source Target Expected result Result
router pod on compute-0 app pod on compute-0 PASS PASS
router pod on compute-2 app pod on compute-0 PASS FAIL

Playing around with hostnetwork and sdn

oc new-project hostnetwork
oc create sa hostnetwork
oc adm policy add-scc-to-user hostnetwork -z hostnetwork

oc apply -n hostnetwork -f - <<EOF
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: hostnetwork
spec:
  selector:
    matchLabels:
      name: hostnetwork
  template:
    metadata:
      labels:
        name: hostnetwork
    spec:
      nodeSelector:
        node-role.kubernetes.io/worker: 
      hostNetwork: true
      containers:
      - name: demo-http
        image: quay.io/rbo/demo-http:master
        env:
        - name: MY_NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
EOF


oc apply -n hostnetwork -f - <<EOF
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: sdn
spec:
  selector:
    matchLabels:
      name: sdn
  template:
    metadata:
      labels:
        name: sdn
    spec:
      nodeSelector:
        node-role.kubernetes.io/worker: 
      containers:
      - name: demo-http
        image: quay.io/rbo/demo-http:master
        env:
        - name: MY_NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
EOF


oc label namespace/hostnetwork name=hostnetwork

oc create -n np-test -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-from-hostnetwork
spec:
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          name: hostnetwork
  podSelector: {}
  policyTypes:
  - Ingress
EOF

Check

oc get pods -n np-test -l app=nginx-example  -o wide
NAME                    READY   STATUS    RESTARTS   AGE     IP            NODE        NOMINATED NODE   READINESS GATES
nginx-example-1-ls67j   1/1     Running   0          6d4h    10.131.0.14   compute-0   <none>           <none>
nginx-example-1-ssqnf   1/1     Running   0          3h11m   10.128.0.33   compute-1   <none>           <none>

# pick on pod ip, nodename and fill in below

oc project hostnetwork


oc get pods -o name | while read line ; do echo 'echo -n "$HOSTNAME: Connect from $MY_NODE_NAME to TARGET_POD_NODE_NAME" ; curl -s --connect-timeout 1 -I http://TARGET_POD_IP:8080/  >/dev/null && echo "OK" || echo "FAIL"; '| oc rsh $line; done

Output:

compute-2: Connect from compute-2 to compute-1 FAIL
master-2: Connect from master-2 to compute-1 FAIL
master-0: Connect from master-0 to compute-1 FAIL
compute-0: Connect from compute-0 to compute-1 FAIL
compute-1: Connect from compute-1 to compute-1 OK
master-1: Connect from master-1 to compute-1 FAIL
sdn-4tfww: Connect from compute-1 to compute-1 OK
sdn-p88xb: Connect from master-2 to compute-1 OK
sdn-pv2rz: Connect from compute-0 to compute-1 OK
sdn-tfzj2: Connect from compute-2 to compute-1 OK
sdn-tngpc: Connect from master-1 to compute-1 OK
sdn-z276l: Connect from master-0 to compute-1 OK
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment