zerotier 6PLANE, docker, and consul

docker-compose.yml

# you need to set ZT6PLANE to match your network

version: '2.3'

networks:
  zerotier:
    driver: bridge
    enable_ipv6: true
    internal: false
    ipam:
      config:
        - subnet: $ZT6PLANE::/80

volumes:
  zerotier_var:
  consul_data_a:
  consul_data_b:
  consul_data_c:

services:
  zerotier:
    image: zerotier/zerotier-containerized
    devices:
      - /dev/net/tun
    network_mode: host
    cap_add:
      - NET_ADMIN
      - SYS_ADMIN
    volumes:
      - zerotier_var:/var/lib/zerotier-one/

  consul_a:
    image: consul:1.0.0
    hostname: consul-a
    command:
      agent
      -bind=$ZT6PLANE::1234:a
      -bootstrap-expect=3
      -recursor=127.0.0.11
      -server
      -ui
    networks:
      zerotier:
        ipv6_address: $ZT6PLANE::1234:a
    expose:  # https://www.consul.io/docs/agent/options.html#ports-used
      - "8300/tcp"  # server RPC
      - "8301/tcp"  # serf LAN
      - "8301/udp"  # serf LAN
      - "8302/tcp"  # serf WAN
      - "8302/udp"  # serf WAN
      - "8500/tcp"  # http
    volumes:
      - consul_data_a:/consul/data

  consul_b:
    image: consul:1.0.0
    hostname: consul-b
    command:
      agent
      -bind=$ZT6PLANE::1234:b
      -bootstrap-expect=3
      -recursor=127.0.0.11
      -retry-join [$ZT6PLANE::1234:a] -retry-join [$ZT6PLANE::1234:c]
      -server
    networks:
      zerotier:
        ipv6_address: $ZT6PLANE::1234:b
    expose:  # https://www.consul.io/docs/agent/options.html#ports-used
      - "8300/tcp"  # server RPC
      - "8301/tcp"  # serf LAN
      - "8301/udp"  # serf LAN
      - "8302/tcp"  # serf WAN
      - "8302/udp"  # serf WAN
    volumes:
      - consul_data_b:/consul/data

  consul_c:
    image: consul:1.0.0
    hostname: consul-c
    command:
      agent
      -bind=$ZT6PLANE::1234:c
      -bootstrap-expect=3
      -recursor=127.0.0.11
      -retry-join [$ZT6PLANE::1234:a] -retry-join [$ZT6PLANE::1234:b]
      -server
    networks:
      zerotier:
        ipv6_address: $ZT6PLANE::1234:c
    expose:  # https://www.consul.io/docs/agent/options.html#ports-used
      - "8300/tcp"  # server RPC
      - "8301/tcp"  # serf LAN
      - "8301/udp"  # serf LAN
      - "8302/tcp"  # serf WAN
      - "8302/udp"  # serf WAN
    volumes:
      - consul_data_c:/consul/data

On my docker-for-mac install, this works as expected.

On my Fedora install with Firewalld and a similar docker-compose file (but with vault and a few other things), I get these errors with docker logs -f shared_consul-a_1:

    2017/10/22 05:57:43 [INFO] serf: EventMemberJoin: consul-b fcf0:a9af:174b:40c3:ff2d:0:46fa:2
    2017/10/22 05:57:43 [INFO] consul: Adding LAN server consul-b (Addr: tcp/[fcf0:a9af:174b:40c3:ff2d:0:46fa:2]:8300) (DC: dc1)
    2017/10/22 05:57:44 [WARN] consul: error getting server health from "consul-b": last request still outstanding
    2017/10/22 05:57:44 [WARN] consul: error getting server health from "consul-b": rpc error getting client: failed to get conn: dial tcp [fcf0:a9af:174b:40c3:ff2d:0:46fa:1]:0->[fcf0:a9af:174b:40c3:ff2d:0:46fa:2]:8300: i/o timeout
    2017/10/22 05:57:47 [WARN] consul: error getting server health from "consul-b": context deadline exceeded
    2017/10/22 05:57:48 [WARN] consul: error getting server health from "consul-b": last request still outstanding
    2017/10/22 05:57:49 [INFO] memberlist: Suspect consul-b has failed, no acks received
    2017/10/22 05:57:50 [WARN] consul: error getting server health from "consul-b": last request still outstanding
    2017/10/22 05:57:52 [WARN] consul: error getting server health from "consul-b": last request still outstanding
    2017/10/22 05:57:52 [ERR] memberlist: Push/Pull with html-zt-stytt-com-56e95beb1e78 failed: dial tcp [fcf0:a9af:174b:40c3:ff2d::8]:8301: getsockopt: permission denied
    2017/10/22 05:57:54 [WARN] consul: error getting server health from "consul-b": last request still outstanding
    2017/10/22 05:57:55 [INFO] memberlist: Suspect consul-b has failed, no acks received
    2017/10/22 05:57:56 [WARN] consul: error getting server health from "consul-b": last request still outstanding
    2017/10/22 05:58:10 [WARN] consul: error getting server health from "consul-b": last request still outstanding
    2017/10/22 05:58:11 [INFO] memberlist: Suspect html-stitthappens-com-69ba29a3ede4 has failed, no acks received

If I run consul members the other nodes flap between alive and failed. It must be something with firewalld. From my laptop nmap shows 8300 open on all these hosts, but between each other it shows as stealth.

A google search for getsockopt: permission denied doesn't even return a full page of results. I might just ditch Fedora but I'd like to figure this out.