vagrantup: multi-VM with NAT router

0_target

vagrant@mgr-1:~$ sudo docker node ls
ID                            HOSTNAME   STATUS    AVAILABILITY   MANAGER STATUS   ENGINE VERSION
iiwwj9kkinfenbajt5zgzirmq *   mgr-1      Ready     Active         Leader           28.0.4
d8f1nruuyjf8uswzukd9o9m56     mgr-2      Ready     Active         Reachable        28.0.4
v2o8d8n5glx36pjrplaxjjb07     mgr-3      Ready     Active         Reachable        28.0.4
43cvywnbgsewn2wxxn63ttanz     wrk-1      Ready     Active                          28.0.4
ru1s8lx4r75rc28ks7v4hsqj5     wrk-2      Ready     Active                          28.0.4
5ptlyjg465c9c7hx98h69coks     wrk-3      Ready     Active                          28.0.4

1_Vagrantfile

# plugins enforcement
unless Vagrant.has_plugin?("vagrant-hostmanager")
  system("vagrant plugin install vagrant-hostmanager")
  puts "dependencies installed, please run again"
  exit
end

# inline provisioning that has to run before `ansible`
$nodenet=<<INLINE
 ip route delete default
 ip route add default via $1
 echo 'alias p1="ping -c 3 1.1.1.1"' >> .bash_aliases
INLINE

$gwnat=<<INLINE
 echo 'net.ipv4.conf.all.forwarding=1' >> /etc/sysctl.conf
 sysctl -p
 cat >> /etc/nftables.conf <<NFT
 table ip nat {
        chain postrouting {
                type nat hook postrouting priority srcnat;
                policy accept;
                ip saddr 172.23.1.0/24 oifname eth0 masquerade;
                ip saddr 10.42.1.0/24 oifname eth0 masquerade;
        }
 }
NFT
 systemctl enable --now nftables > /dev/null 2>&1
INLINE
#NAT

$preansible=<<INLINE
 touch .hushlogin
 export DEBIAN_FRONTEND=noninteractive
 apt-get -y install tcpdump
INLINE


$mgr=<<INLINE
 [ -z "$1" ] || echo $1 > /tmp/mgr-inline.out
INLINE
$wrk=<<INLINE
 [ -z "$1" ] || echo $1 > /tmp/wrk-inline.out
INLINE

Vagrant.configure("2") do |config|
  config.vm.box = "bento/debian-12.9"
  config.vm.provider "virtualbox" do |v|
    v.memory = 512
  end
  # create a VM's /etc/host; settings are to include all addresses independent
  # of VMs already created or not (or offline at the moment)
  config.hostmanager.enabled = true
  config.hostmanager.manage_host = false # do not mess with laptop
  config.hostmanager.guest_host = true
  config.hostmanager.include_offline = true
  config.hostmanager.ignore_private_ip = false # to get private_network resolving

  # route between mgr+wrk and outbound incl NAT
  config.vm.define "gateway" do |gw|
    gw.vm.network "private_network", ip: "172.23.1.23", netmask: "255.255.255.0"
    gw.vm.network "private_network", ip: "10.42.1.42", netmask: "255.255.255.0"
    gw.vm.synced_folder "./pcap/gateway", "/home/vagrant/pcap", create: true
    gw.vm.hostname = "GateWay"
    gw.vm.provider "virtualbox" do |v|
      v.customize ["modifyvm", :id, "--memory", 1024]
    end
    gw.vm.provision "shell", inline: $gwnat
    gw.vm.provision "shell", inline: $preansible
    gw.vm.provision "ansible" do |aa|
      aa.compatibility_mode = "2.0"
      aa.playbook = "aa/gateway.yml"
      aa.extra_vars = "aa/host_vars/all"
      aa.groups = {
       "gw" => ["gateway"]
      }
    end
  end

  # how many nodes for manager and workers
  mgr_c = 3
  wrk_c = 3
  # swarm manager nodes ; alt. `(1..count).each `
  ([ "mgr" ] * mgr_c).each_with_index do |mgr, i|
    config.vm.define "#{mgr}-#{i+1}" do |mgrcfg|
      mgrcfg.vm.network "private_network", ip: "172.23.1.10#{i+1}", netmask: "255.255.255.0"
      # only need to port forward to one node as we use swarm overlay, but
      # since this is a loop, just add some ports above. no need on "wrk" at least
      mgrcfg.vm.network "forwarded_port",
        host: "924#{i}", guest: "9240", guest_ip: "172.23.1.10#{i+1}"
      mgrcfg.vm.network "forwarded_port",
        host: "333#{i}", guest: "3333", guest_ip: "172.23.1.10#{i+1}"
      mgrcfg.vm.synced_folder "./pcap/#{mgr}-#{i+1}", "/home/vagrant/pcap", create: true
      mgrcfg.vm.hostname = "#{mgr}-#{i+1}"
      mgrcfg.vm.provision "shell", inline: $nodenet, args: "172.23.1.23"
      mgrcfg.vm.provision "shell", inline: $mgr, args: "custom here"
      mgrcfg.vm.provision "shell", inline: $preansible
      mgrcfg.vm.provision "ansible" do |aa|
        aa.compatibility_mode = "2.0"
        aa.verbose = "v"
        aa.playbook = "aa/mgr.yml"
        aa.extra_vars = "aa/host_vars/all"
        aa.host_vars = {
          "mgr-#{i+1}" => {
            "main_ip" => "172.23.1.10#{i+1}",
          }
        }
        aa.groups = {
          "mgr" => [ "mgr-[1:#{mgr_c}]" ],
          "wrk" => [ "wrk-[1:#{wrk_c}]" ]
        }
      end
    end
  end
  # swarm worker nodes
  ([ "wrk" ] * wrk_c).each_with_index do |wrk, i|
    config.vm.define "#{wrk}-#{i+1}" do |wrkcfg|
      wrkcfg.vm.network "private_network", ip: "10.42.1.10#{i+1}", netmask: "255.255.255.0"
      wrkcfg.vm.synced_folder "./pcap/#{wrk}-#{i+1}", "/home/vagrant/pcap", create: true
      wrkcfg.vm.hostname = "#{wrk}-#{i+1}"
      wrkcfg.vm.provision "shell", inline: $nodenet, args: "10.42.1.42"
      wrkcfg.vm.provision "shell", inline: $wrk, args: "custom here"
      wrkcfg.vm.provision "shell", inline: $preansible
      wrkcfg.vm.provision "ansible" do |aa|
        aa.compatibility_mode = "2.0"
        aa.verbose = "vv"
        aa.playbook = "aa/wrk.yml"
        aa.extra_vars = "aa/host_vars/all"
        aa.host_vars = { "wrk-#{i+1}" => {"main_ip" => "10.42.1.10#{i+1}"} }
        aa.groups = {
          "mgr" => [ "mgr-[1:#{mgr_c}]" ],
          "wrk" => [ "wrk-[1:#{wrk_c}]" ]
        }
      end
    end
  end
end

gateway.yml

---
- name: ansible for gateways
  hosts: gw
  become: true

  tasks:
  - name: install something
    ansible.builtin.package:
      name:
        - jq

groupvars.j2

{% for node in groups["mgr"] %}
Found manager in this group: {{ node }}
{% endfor %}
{% for node in groups["wrk"] %}
Found worker in this group: {{ node }}
{% endfor %}

mgr.yml

---
- name: prereqs and docker-ce for manager nodes
  hosts: mgr
  become: true

  pre_tasks:
    - name: install something
      ansible.builtin.package:
        name:
          - jq
    - name: unroll group
      ansible.builtin.template:
        src: groupvars.j2
        dest: "/tmp/aa_group"

  roles:
    - role: docker-ce
    
- name: dockerswarm adaptions
  hosts: mgr
  become: true

  roles:
    - role: docker-swarm
    - role: dockerswarm-dashboard
    - role: dockerswarm-webapp

roles_docker-ce_tasks_main.yml

- name: Create apt keyring directory
  # very minimal debootstraps might not have it
  ansible.builtin.file:
    path: "/etc/apt/keyrings"
    state: directory
    owner: root
    mode: 0755

- name: Install GPG key from docker.com
  # no need to dearmor since debian9
  ansible.builtin.get_url:
    url: https://download.docker.com/linux/debian/gpg
    dest: /etc/apt/keyrings/docker.asc

- name: Docker repo in sources
  ansible.builtin.apt_repository:
    repo: |
      deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc]
      https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable
    state: present
    filename: docker

- name: Install docker-ce and python3 packages
  ansible.builtin.apt:
    name:
      - docker-ce
      - docker-ce-cli
      - docker-compose-plugin
      - docker-ce-rootless-extras
      - python3-docker
    update_cache: true

- name: Create docker daemon config
  ansible.builtin.template:
    src: daemon.json.j2
    dest: "/etc/docker/daemon.json"

- name: Restart docker for metrics
  ansible.builtin.service:
    name: docker
    state: restarted