rkt-flannel-cluster

bridge-nat1.conf

{
    "name": "bridge-nat1",
    "type": "bridge",
    "bridge": "brnat1",
    "isDefaultGateway": true,
    "ipam": {
        "type": "host-local",
        "subnet": "10.2.1.0/24"
    }
}

bridge-nat2.conf

{
    "name": "bridge-nat2",
    "type": "bridge",
    "bridge": "brnat2",
    "isDefaultGateway": true,
    "ipam": {
        "type": "host-local",
        "subnet": "10.2.2.0/24"
    }
}

goreman.log

15:41:51 flannel2 | Starting flannel2 on port 5003
15:41:51 flannel1 | Starting flannel1 on port 5002
15:41:51     etcd | Starting etcd on port 5000
15:41:51  etcdctl | Starting etcdctl on port 5001
15:41:51     etcd | image: using image from file /nix/store/2v22yr7jayv5rlj3zrnf0p0qq0x5l96d-rkt-1.11.0/lib/rkt/stage1-images/stage1-coreos.aci
15:41:53     etcd | image: using image from file ../ACIs/etcd-2.3.7-linux-amd64.aci
15:41:54     etcd | image: signature verified:
15:41:54     etcd |   Stefan Junker <[email protected]>
15:41:54     etcd |   Stefan Junker <[email protected]>
15:41:56 flannel2 | image: using image from local store for image name coreos.com/rkt/stage1-coreos:1.11.0
15:41:56 flannel1 | image: using image from local store for image name coreos.com/rkt/stage1-coreos:1.11.0
15:41:56 flannel2 | image: using image from file /home/steveej/src/github/steveej/nix2aci/ACIs/busybox-1.23.2-linux-amd64.aci
15:41:56 flannel1 | image: using image from file /home/steveej/src/github/steveej/nix2aci/ACIs/busybox-1.23.2-linux-amd64.aci
15:41:57 flannel2 | image: signature verified:
15:41:57 flannel2 |   Stefan Junker <[email protected]>
15:41:57 flannel2 |   Stefan Junker <[email protected]>
15:41:57 flannel1 | image: signature verified:
15:41:57 flannel1 |   Stefan Junker <[email protected]>
15:41:57 flannel1 |   Stefan Junker <[email protected]>
15:42:10     etcd | run: group "rkt" not found, will use default gid when rendering images
15:42:10     etcd | networking: loading networks from /etc/rkt/net.d
15:42:10     etcd | networking: loading network default-restricted with type ptp
15:42:10     etcd | stage1: warning: error setting journal ACLs, you'll need root to read the pod journal: group "rkt" not found
15:42:10     etcd | stage1: warning: no volume specified for mount point "datadir", implicitly creating an "empty" volume. This volume will be removed when the pod is garbage-collected.
15:42:10     etcd | stage1: warning: no volume specified for mount point "resolvconf", implicitly creating an "empty" volume. This volume will be removed when the pod is garbage-collected.
15:42:10     etcd | stage1: warning: no volume specified for mount point "datadir", implicitly creating an "empty" volume. This volume will be removed when the pod is garbage-collected.
15:42:10     etcd | stage1: warning: no volume specified for mount point "resolvconf", implicitly creating an "empty" volume. This volume will be removed when the pod is garbage-collected.
15:42:12  etcdctl | { "Network": "172.15.0.0/16", "Backend": { "Type": "vxlan" } }
15:42:12  etcdctl | Terminating etcdctl
15:42:13 flannel1 | run: group "rkt" not found, will use default gid when rendering images
15:42:13 flannel1 | networking: loading networks from /etc/rkt/net.d
15:42:13 flannel1 | networking: loading network bridge-nat1 with type bridge
15:42:13 flannel1 | networking: loading network default-restricted with type ptp
15:42:13 flannel1 | stage1: warning: error setting journal ACLs, you'll need root to read the pod journal: group "rkt" not found
15:42:13 flannel1 | [20056.677715] busybox[5]: I0726 22:42:13.876486 00005 main.go:126] Installing signal handlers
15:42:13 flannel1 | [20056.678283] busybox[5]: I0726 22:42:13.877128 00005 manager.go:133] Determining IP address of default interface
15:42:13 flannel1 | [20056.678826] busybox[5]: I0726 22:42:13.877671 00005 manager.go:163] Using 10.2.1.3 as external interface
15:42:13 flannel1 | [20056.679110] busybox[5]: I0726 22:42:13.877693 00005 manager.go:164] Using 10.2.1.3 as external endpoint
15:42:13 flannel1 | [20056.681850] busybox[5]: I0726 22:42:13.880679 00005 local_manager.go:179] Picking subnet in range 172.15.1.0 ... 172.15.255.0
15:42:13 flannel1 | [20056.683485] busybox[5]: I0726 22:42:13.882308 00005 manager.go:246] Lease acquired: 172.15.70.0/24
15:42:13 flannel1 | [20056.684575] busybox[5]: I0726 22:42:13.883386 00005 network.go:58] Watching for L3 misses
15:42:13 flannel1 | [20056.684987] 
15:42:13 flannel1 | busybox[5]: I0726 22:42:13.883834 00005 network.go:66] Watching for new subnet leases
15:42:15 flannel2 | run: group "rkt" not found, will use default gid when rendering images
15:42:15 flannel2 | networking: loading networks from /etc/rkt/net.d
15:42:15 flannel2 | networking: loading network bridge-nat2 with type bridge
15:42:15 flannel2 | networking: loading network default-restricted with type ptp
15:42:15 flannel2 | stage1: warning: error setting journal ACLs, you'll need root to read the pod journal: group "rkt" not found
15:42:15 flannel2 | [20058.584471] busybox[5]: I0726 22:42:15.783266 00005 main.go:126] Installing signal handlers
15:42:15 flannel2 | [20058.584722] busybox[5]: I0726 22:42:15.783565 00005 manager.go:133] Determining IP address of default interface
15:42:15 flannel2 | [20058.584909] busybox[5]: I0726 22:42:15.783762 00005 manager.go:163] Using 10.2.2.2 as external interface
15:42:15 flannel2 | [20058.585177] busybox[5]: I0726 22:42:15.783912 00005 manager.go:164] Using 10.2.2.2 as external endpoint
15:42:15 flannel2 | [20058.586886] busybox[5]: I0726 22:42:15.785731 00005 local_manager.go:179] Picking subnet in range 172.15.1.0 ... 172.15.255.0
15:42:15 flannel1 | [20058.587519] busybox[5]: I0726 22:42:15.786365 00005 network.go:153] Handling initial subnet events
15:42:15 flannel1 | 
15:42:15 flannel2 | [20058.587537] busybox[5]: I0726 22:42:15.786384 00005 manager.go:246] Lease acquired: 172.15.28.0/24
15:42:15 flannel1 | [20058.587903] busybox[5]: I0726 22:42:15.786740 00005 device.go:161] calling GetL2List() dev.link.Index: 6
15:42:15 flannel2 | [20058.588065] busybox[5]: I0726 22:42:15.786657 00005 network.go:58] Watching for L3 misses
15:42:15 flannel1 | [20058.588214] busybox
15:42:15 flannel1 | [5]: I0726 22:42:15.786824 00005 device.go:166] calling NeighAdd: 10.2.2.2, 7a:31:d5:64:0a:b5
15:42:15 flannel2 | [20058.588458] busybox[5]: I0726 22:42:15.786683 00005 network.go:66] Watching for new subnet leases
15:42:15 flannel2 | [20058.588705] busybox[5]: I0726 22:42:15.787556 00005 network.go:153] Handling initial subnet events
15:42:15 flannel2 | 
15:42:15 flannel2 | [20058.588972] busybox[5]: I0726 22:42:15.787569 00005 device.go:161] calling GetL2List() dev.link.Index: 6
15:42:15 flannel2 | [20058.589194] busybox[5]: I0726 22:42:15.787628 00005 device.go:166] calling NeighAdd: 10.2.1.3, ae:b5:6d:fc:3d:61

Procfile

# Use goreman to run `go get github.com/mattn/goreman`
etcd: sudo rkt run --net=default-restricted:IP=172.16.28.2 --port=etcd2:2379  ../ACIs/etcd-2.3.7-linux-amd64.aci -- --listen-peer-urls 'http://0.0.0.0:2380,http://0.0.0.0:7001' --listen-client-urls 'http://0.0.0.0:2379,http://0.0.0.0:4001' --initial-advertise-peer-urls 'http://172.16.28.2:2380,http://172.16.28.2:7001' --advertise-client-urls 'http://172.16.28.2:2379,http://172.16.28.2:4001' --initial-cluster="default=http://172.16.28.2:2380,default=http://172.16.28.2:7001"
etcdctl: while true; do sleep 1; etcdctl --endpoints=http://172.16.28.2:2379 --no-sync set /coreos.com/network/config  '{ "Network": "172.15.0.0/16", "Backend": { "Type": "vxlan" } }' 2>/dev/null && exit 0; done
flannel1: sleep 5; sudo rkt run --net=bridge-nat1,default-restricted --volume flanneld,kind=host,source=/home/steveej/src/github/coreos/flannel/bin/flanneld --mount volume=flanneld,target=/bin/flanneld --volume nix,kind=host,source=/nix/store,readOnly=true --mount volume=nix,target=/nix/store /home/steveej/src/github/steveej/nix2aci/ACIs/busybox-1.23.2-linux-amd64.aci --exec /bin/flanneld -- --etcd-endpoints=http://172.16.28.2:2379
flannel2: sleep 5; sudo rkt run --net=bridge-nat2,default-restricted --volume flanneld,kind=host,source=/home/steveej/src/github/coreos/flannel/bin/flanneld --mount volume=flanneld,target=/bin/flanneld --volume nix,kind=host,source=/nix/store,readOnly=true --mount volume=nix,target=/nix/store /home/steveej/src/github/steveej/nix2aci/ACIs/busybox-1.23.2-linux-amd64.aci --exec /bin/flanneld -- --etcd-endpoints=http://172.16.28.2:2379