Testcase for IPVS over IPSec

Vagrantfile.rb

require 'digest/md5'
require 'base64'

# Using a static token to simplify automation
$kube_master = "scheduler"
$kube_token = "aaaaaa.bbbbbbbbbbbbbbbb"
$kube_cidr = "192.168.0.0/16"
$hosts = {
  "scheduler" => "172.16.0.2",
  "worker" => "172.16.0.4"
}

$kubeadm_tpl = <<-ADM
apiVersion: kubeadm.k8s.io/v1beta1
kind: InitConfiguration
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: %{kube_token}
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
localAPIEndpoint:
  advertiseAddress: %{ip}
---
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
networking:
  podSubnet: %{kube_cidr}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
ADM

$ike_tpl = <<-IKE
    %{to_host} {
        version = 2
        local_addrs = %{from_ip}
        remote_addrs = %{to_ip}
        unique = replace

        local {
            auth = psk
            id = %{from_host}
        }

        remote {
            auth = psk
            id = %{to_host}
        }

        children {
            %{to_host} {
                mode = transport
                esp_proposals = null-sha256
                inactivity = 1800s
                start_action = trap
                dpd_action = trap
                close_action = trap
            }
        }

        reauth_time = 24h
        rekey_time = 4h
        mobike = no
    }
IKE

$psk_tpl = <<-PSK
    ike-%{host} {
        id = %{host}
        secret = %{secret}
    }
PSK

def ipsec_conf(host)
  ikes = ""
  psks = ""
  from_host = host
  from_ip = $hosts[host]
  $hosts.each do |name, ip|
    if name != host
      to_host = name
      to_ip = $hosts[name]
      ikes += $ike_tpl % {from_host: host, from_ip: from_ip, to_host: to_host, to_ip: to_ip}
    end
    secret = Digest::MD5.hexdigest(name)
    psks += $psk_tpl % {host: name, secret: secret}
  end

  return "connections {
#{ikes}
}

secrets {
#{psks}
}"
end

Vagrant.configure("2") do |config|
  config.vm.box = "bento/ubuntu-16.04"

  config.vm.provider "parallels" do |prl|
    prl.memory = 2048
    prl.cpus = 2
  end

  config.vm.provider "virtualbox" do |v|
    v.memory = 2048
    v.cpus = 2
  end

  config.vm.provision "software", type: "shell", privileged: true, inline: <<-SHELL
    apt-get update

    # Disable swap
    sed -e '/swap/ d' -i /etc/fstab
    swapoff -a

    # Install strongswan, in VICI mode
    add-apt-repository -u ppa:topdog/strongswan
    apt-get install -y charon-systemd

    # Install docker, pinned to 18.06
    curl -fsSL https://get.docker.com -o get-docker.sh
    VERSION=18.06 sh get-docker.sh
    apt-mark hold docker-ce

    # Install kubeadm and friends
    curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
    add-apt-repository -u 'deb https://apt.kubernetes.io/ kubernetes-xenial main'
    apt-get install -y kubelet kubeadm kubectl ipvsadm
    apt-mark hold kubelet kubeadm kubectl

    # Enable IPVS modules
    for mod in ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh; do
      modprobe $mod
      echo $mod >> /etc/modules
    done

    # Set the default forwarding policy to accept, just in case
    iptables -P FORWARD ACCEPT
  SHELL

  $hosts.each do |name, ip|
    config.vm.define name do |box|
      box.vm.network "private_network", ip: ip
      box.vm.hostname = name

      # Annoyingly, we can't use a temp file because provisioners are run after the file is evaluated
      # Instead, base64 the string and strip lines, so that we can send it in an env var
      ipsec_data = Base64.encode64(ipsec_conf(name)).gsub("\n", "")
      box.vm.provision "ipsec", type: "shell", privileged: true, env: {data: ipsec_data}, inline: <<-SHELL
        mkdir -p /etc/swanctl/conf.d
        echo $data | base64 -d > /etc/swanctl/conf.d/ipsec.conf
        systemctl reload strongswan-swanctl
      SHELL

      if name == $kube_master
        kubeadm_cfg = $kubeadm_tpl % {kube_token: $kube_token, kube_cidr: $kube_cidr, ip: ip}
        kubeadm_data = Base64.encode64(kubeadm_cfg).gsub("\n", "")
        box.vm.provision "kube_master", type: "shell", privileged: true, env: {cfg: kubeadm_data}, args: [ip], inline: <<-SHELL
          echo $cfg | base64 -d > /tmp/kubeadm.yaml
          kubeadm init --config /tmp/kubeadm.yaml
          echo KUBELET_EXTRA_ARGS=\"--node-ip=$1\" > /etc/default/kubelet
          systemctl restart kubelet
          # KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
        SHELL
      else
        box.vm.provision "kube_node", type: "shell", privileged: true, args: [ip, $kube_token, $hosts[$kube_master]], inline: <<-SHELL
          kubeadm join --token=$2 --discovery-token-unsafe-skip-ca-verification $3:6443
          echo KUBELET_EXTRA_ARGS=\"--node-ip=$1\" > /etc/default/kubelet
          systemctl restart kubelet
        SHELL
      end

      box.vm.provision "kube_reset", type: "shell", privileged: true, run: "never", inline: <<-SHELL
        kubeadm reset -f
      SHELL
    end
  end
end