OpenShift 4.0 installer on GCP

create-libvirt.sh

#/bin/bash

NAME="$1"
if [ -z "$NAME" ]; then
  echo "usage: create-libvirt.sh <name>"
  exit 1
fi

CLUSTER_DIR="$HOME/clusters/${NAME}"
if [ -d "$CLUSTER_DIR" ]; then
  echo "WARNING: cluster ${NAME} already exists at ${CLUSTER_DIR}"
fi

export PATH="$PATH:$PWD/bin"

export OPENSHIFT_INSTALL_BASE_DOMAIN=testing
export OPENSHIFT_INSTALL_CLUSTER_NAME=$NAME
export [email protected]
export OPENSHIFT_INSTALL_PASSWORD="$(<$HOME/tools/admin-password.txt)"
export OPENSHIFT_INSTALL_PLATFORM=libvirt
export OPENSHIFT_INSTALL_SSH_PUB_KEY_PATH=$HOME/.ssh/id_rsa.pub
export OPENSHIFT_INSTALL_PULL_SECRET_PATH=$HOME/tools/quay-pull-secret.json
export OPENSHIFT_INSTALL_LIBVIRT_URI="qemu+tcp://192.168.122.1/system"
export OPENSHIFT_INSTALL_LIBVIRT_IMAGE="file://$HOME/rhcos-qemu.qcow2"
openshift-install create cluster --log-level=debug --dir="$CLUSTER_DIR" 2>&1 | tee /tmp/installer.log

provision.sh

#!/bin/bash
set -e
set -u
set -o pipefail

# :-)
sudo setenforce 0

# Enable the latest libvirt packages
sudo bash -c 'cat > /etc/yum.repos.d/sig-virt.repo' << EOF
[sig-virt-libvirt-latest]
name=SIG-Virt libvirt packages for CentOS 7 x86_64
baseurl=http://mirror.centos.org/centos-7/7/virt/x86_64/libvirt-latest
enabled=1
EOF

# Enable the latest QEMU packages
sudo bash -c 'cat > /etc/yum.repos.d/sig-virt-kvm-common.repo' << EOF
[sig-virt-kvm-common]
name=SIG-Virt libvirt packages for CentOS 7 x86_64
baseurl=http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common
enabled=1
EOF

# Enable the latest Go packages
sudo rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO
curl -s https://mirror.go-repo.io/centos/go-repo.repo | sudo tee /etc/yum.repos.d/go-repo.repo

# TODO: find the GPG key for SIG-Virt stuff
sudo yum install -y --nogpg libvirt libvirt-devel libvirt-client git golang libvirt-daemon-kvm qemu-kvm bind-utils

# Enable IP forwarding
# https://github.com/openshift/installer/blob/master/docs/dev/libvirt-howto.md#enable-ip-forwarding
sudo sysctl net.ipv4.ip_forward=1
echo "net.ipv4.ip_forward = 1" | sudo tee /etc/sysctl.d/99-ipforward.conf
sudo sysctl -p /etc/sysctl.d/99-ipforward.conf

# Enable non-root access to libvirt stuff
# https://github.com/openshift/installer/blob/master/docs/dev/libvirt-howto.md#make-sure-you-have-permissions-for-qemusystem
sudo bash -c 'cat > /etc/polkit-1/rules.d/80-libvirt.rules' << EOF
polkit.addRule(function(action, subject) {
  if (action.id == "org.libvirt.unix.manage" && subject.local && subject.active && subject.isInGroup("$USER")) {
      return polkit.Result.YES;
  }
});
EOF
sudo usermod --append --groups libvirt $USER

# Configure libvirt to accept TCP connections
# https://github.com/openshift/installer/blob/master/docs/dev/libvirt-howto.md#configure-libvirt-to-accept-tcp-connections
sudo bash -c 'cat >> /etc/libvirt/libvirtd.conf' << EOF
listen_tls = 0
listen_tcp = 1
auth_tcp="none"
tcp_port = "16509"
EOF
sudo bash -c 'cat >> /etc/sysconfig/libvirtd' << EOF
LIBVIRTD_ARGS="--listen"
EOF
sudo bash -c 'cat >> /etc/modprobe.d/kvm.conf' << EOF
options kvm_intel nested=1
EOF
# Ensure nesting is enabled in the kernel
# TODO: verify this is still necessary
sudo modprobe -r kvm_intel
sudo modprobe kvm_intel nested=1
sudo systemctl restart libvirtd
# Set up iptables and firewalld
# TODO: discover the ports
sudo iptables -I INPUT -p tcp -s 192.168.122.0/24 -d 192.168.122.1 --dport 16509 -j ACCEPT -m comment --comment "Allow insecure libvirt clients"
sudo firewall-cmd --zone=trusted --add-source=192.168.126.0/24
sudo firewall-cmd --zone=trusted --add-port=16509/tcp

# Enable NetworkManager DNS overlay
# https://github.com/openshift/installer/blob/master/docs/dev/libvirt-howto.md#set-up-networkmanager-dns-overlay
sudo sed -i -z 's/\[main\]\n/\[main\]\ndns=dnsmasq\n/' /etc/NetworkManager/NetworkManager.conf
echo server=/testing/192.168.126.1 | sudo tee /etc/NetworkManager/dnsmasq.d/tectonic.conf
sudo systemctl restart NetworkManager

# Configure the default libvirt storage pool
# https://github.com/openshift/installer/blob/master/docs/dev/libvirt-howto.md#configure-default-libvirt-storage-pool
sudo virsh pool-define /dev/stdin <<EOF
<pool type='dir'>
  <name>default</name>
  <target>
    <path>/var/lib/libvirt/images</path>
  </target>
</pool>
EOF
sudo virsh pool-start default
sudo virsh pool-autostart default

# Set up terraform
# https://github.com/openshift/installer/blob/master/docs/dev/libvirt-howto.md#install-the-terraform-provider
# https://github.com/openshift/installer/blob/master/docs/dev/libvirt-howto.md#cache-terrafrom-plugins-optional-but-makes-subsequent-runs-a-bit-faster
cat <<EOF > $HOME/.terraformrc
plugin_cache_dir = "$HOME/.terraform.d/plugin-cache"
EOF
GOBIN=~/.terraform.d/plugins go get -u github.com/dmacvicar/terraform-provider-libvirt

# Generate an SSH key for VMs
ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa

# Download an RHCOS image
gsutil cp gs://rhcos/rhcos-qemu.qcow2.gz $HOME
cd $HOME
gunzip $HOME/rhcos-qemu.qcow2.gz

# Download oc
cd $HOME
curl -OL https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz
tar -zxf openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz
sudo mv $HOME/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit/oc /usr/local/bin

# Build the installer
git clone https://github.com/openshift/installer.git $HOME/go/src/github.com/openshift/installer
cd $HOME/go/src/github.com/openshift/installer
hack/get-terraform.sh
TAGS=libvirt_destroy hack/build.sh

# Place the installer and terraform on PATH
sudo ln -sf $HOME/go/src/github.com/openshift/installer/bin/openshift-install /usr/bin/openshift-install
sudo ln -sf $HOME/go/src/github.com/openshift/installer/bin/terraform /usr/bin/terraform

# Build origin
# git clone https://github.com/openshift/origin.git $HOME/go/src/github.com/openshift/origin
# cd $HOME/go/src/github.com/openshift/origin
# go build -o $HOME/go/src/github.com/openshift/installer/bin/oc ./cmd/oc

setup.sh

gcloud compute instances create my-instance \
  --zone us-east1-c --image nested-centos7 \
  --machine-type n1-standard-8 --min-cpu-platform "Intel Haswell" \
  --boot-disk-type pd-ssd --boot-disk-size 256GB

gcloud compute scp --recurse tools my-instance:~/tools