Skip to content

Instantly share code, notes, and snippets.

@yoshikado

yoshikado/deploy-offline-cdk-with-lxd.sh

Last active September 17, 2018 11:22
Show Gist options
  • Save yoshikado/acfc682f36ec6358c113feff8ea3e295 to your computer and use it in GitHub Desktop.
Save yoshikado/acfc682f36ec6358c113feff8ea3e295 to your computer and use it in GitHub Desktop.
Download ZIP
canonical kubernetes offline deployment
Raw
deploy-offline-cdk-with-lxd.sh
#!/usr/bin/env bash
set -e
set -u
## Variables
set_vars(){
APT_MIRROR_HOST="mirror"
LXDKVM_SSTREAM_HOST="mirror"
JUJU_SSTREAM_HOST="mirror"
BOOTSTRAP_NODE_IP=10.10.0.20
CDK_SHRINKWRAP_PATH=$(ls -dt $HOME/cdk-shrinkwrap/*/ | head -1)
PRIV_REGISTRY_HOST="mirror:5000"
MACHINE_IPS="10.10.0.21 \
10.10.0.22 \
10.10.0.23 \
10.10.0.24 \
10.10.0.25 \
10.10.0.26 \
10.10.0.27 \
10.10.0.28 \
10.10.0.29 \
10.10.0.30 \
10.10.0.31"
KUBE_VER="1.11/stable"
ETCD_VER="3.2/stable"
SERIES=xenial
CERT_PATH="/etc/pki/tls/certs/"
CERT_FILE="mirror.crt"
FAN_CONFIG="10.10.0.0/24=252.0.0.0/8"
}
check_pkg(){
if dpkg --get-selections | grep -q "^$1[[:space:]]*install$" >/dev/null; then
return
else
echo "package $1 is missing."
exit 1
fi
}
check_prerequisite(){
check_pkg "jq"
check_pkg "juju-2.0"
}
## Bootstrap juju
juju_bootstrap(){
tee ~/mycloud.yaml > /dev/null << EOL
clouds:
manual:
type: manual
endpoint: $BOOTSTRAP_NODE_IP
EOL
juju add-cloud --replace manual ~/mycloud.yaml
echo "bootstrapping juju..."
ssh-keyscan $BOOTSTRAP_NODE_IP
sleep 2
ssh-keyscan $BOOTSTRAP_NODE_IP > ~/.ssh/known_hosts
juju bootstrap --no-gui \
--bootstrap-series=$SERIES \
--config apt-mirror=http://${APT_MIRROR_HOST}/archive.ubuntu.com/ubuntu/ \
--config agent-stream=release \
--config container-image-metadata-url=https://${LXDKVM_SSTREAM_HOST}/lxdkvm/_latest \
--config agent-metadata-url=https://${JUJU_SSTREAM_HOST}/juju/ \
--debug manual manual-controller > juju-bootstrap.log 2>&1
juju model-config fan-config=$FAN_CONFIG
juju model-config container-networking-method=fan
}
## add machines
juju_add_machines(){
for machine in $MACHINE_IPS
do
ssh-keyscan $machine
sleep 2
ssh-keyscan $machine >> ~/.ssh/known_hosts
juju add-machine --debug ssh:ubuntu@$machine > juju-add-machine-$machine.log 2>&1 &
done
sleep 60
## prepare machines for deploy
len=$(wc -w <<< "$MACHINE_IPS")
for id in $(seq 0 $(expr $len - 1))
do
while [ $(juju status --format json | jq -r ".machines[\"$id\"].\"juju-status\".current") != "started" ]
do
echo "waiting for machine $id to be ready..."
sleep 5
done
echo "machine $id is ready. configuring machine now"
juju scp ${CDK_SHRINKWRAP_PATH}/resources/core.snap $id:
juju scp ${CERT_PATH}${CERT_FILE} $id:
done
juju run --all "sudo snap install --dangerous /home/ubuntu/core.snap"
juju run --all "sudo mv /home/ubuntu/$CERT_FILE /usr/local/share/ca-certificates/"
juju run --all "sudo update-ca-certificates"
# juju won't replace securiy.ubuntu.com to value in --apt-mirror
juju run --all -- sudo sed -i 's/security.ubuntu.com/${APT_MIRROR_HOST}\/archive.ubuntu.com/g' /etc/apt/sources.list
}
# deploy all
juju_deploy(){
# deploy etcd
juju deploy -n 3 --resource etcd=${CDK_SHRINKWRAP_PATH}/resources/etcd/etcd.snap --resource snapshot=${CDK_SHRINKWRAP_PATH}/resources/etcd/snapshot.gz \
--to 0,1,2 ${CDK_SHRINKWRAP_PATH}/charms/etcd --config ~/cdk-config.yaml
# deploy kubernetes-master
juju deploy -n 2 --resource cdk-addons=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/cdk-addons.snap \
--resource kube-apiserver=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-apiserver.snap \
--resource kube-controller-manager=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-controller-manager.snap \
--resource kube-scheduler=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-scheduler.snap \
--resource kubectl=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kubectl.snap --to 3,4 ${CDK_SHRINKWRAP_PATH}/charms/kubernetes-master --config ~/cdk-config.yaml
# deploy flannel
juju deploy --resource flannel-amd64=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-amd64.gz --resource flannel-arm64=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-arm64.gz \
--resource flannel-s390x=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-s390x.gz ${CDK_SHRINKWRAP_PATH}/charms/flannel
# deploy easyrsa
juju deploy -n 1 --resource easyrsa=${CDK_SHRINKWRAP_PATH}/resources/easyrsa/easyrsa.tgz --to lxd:8 ${CDK_SHRINKWRAP_PATH}/charms/easyrsa
# deploy kubernetes-worker
juju deploy -n 5 --resource cni-amd64=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-amd64.tgz --resource cni-arm64=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-arm64.tgz \
--resource cni-s390x=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-s390x.tgz --resource kube-proxy=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kube-proxy.snap \
--resource kubectl=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kubectl.snap --resource kubelet=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kubelet.snap \
--to 5,6,7 ${CDK_SHRINKWRAP_PATH}/charms/kubernetes-worker --config ~/cdk-config.yaml
# deploy ceph-osd
juju deploy -n 3 --to 5,6,7 $CDK_SHRINKWRAP_PATH/charms/ceph-osd
# deploy ceph-mon
juju deploy -n 3 --to lxd:5,lxd:6,lxd:7 $CDK_SHRINKWRAP_PATH/charms/ceph-mon
# deploy kubeapi-load-balancer
juju deploy -n 1 --to 8 ${CDK_SHRINKWRAP_PATH}/charms/kubeapi-load-balancer
# deploy prometheus2
juju deploy -n 1 --resource prometheus=${CDK_SHRINKWRAP_PATH}/resources/prometheus/prometheus.snap --to 9 ${CDK_SHRINKWRAP_PATH}/charms/prometheus2
# deploy grafana
juju deploy -n 1 --to lxd:9 ${CDK_SHRINKWRAP_PATH}/charms/grafana --config ~/cdk-config.yaml
# deploy telegraf
juju deploy ${CDK_SHRINKWRAP_PATH}/charms/telegraf --config ~/cdk-config.yaml
# deploy elasticsearch
juju deploy -n 1 --to lxd:8 ${CDK_SHRINKWRAP_PATH}/charms/elasticsearch --config ~/cdk-config.yaml
# deploy filebeat
juju deploy ${CDK_SHRINKWRAP_PATH}/charms/filebeat --config ~/cdk-config.yaml
# deploy graylog
juju deploy -n 1 --resource graylog=${CDK_SHRINKWRAP_PATH}/resources/graylog/graylog.snap --to 10 ${CDK_SHRINKWRAP_PATH}/charms/graylog
# deploy mongodb
juju deploy -n 1 --to lxd:10 ${CDK_SHRINKWRAP_PATH}/charms/mongodb
# deploy apache2
juju deploy -n 1 --to lxd:10 ${CDK_SHRINKWRAP_PATH}/charms/apache2 --config ~/cdk-config.yaml
}
make_config_yaml(){
tee ~/cdk-config.yaml > /dev/null << EOL
kubernetes-master:
channel: $KUBE_VER
kubernetes-worker:
kubelet-extra-args: "pod-infra-container-image=$PRIV_REGISTRY_HOST/google_containers/pause-amd64:3.1"
docker-opts: "--insecure-registry=$PRIV_REGISTRY_HOST"
nginx-image: "$PRIV_REGISTRY_HOST/nginx-ingress-controller:0.16.1"
default-backend-image: "$PRIV_REGISTRY_HOST/defaultbackend:1.4"
channel: $KUBE_VER
etcd:
channel: $ETCD_VER
apache2:
enable_modules: "headers proxy_html proxy_http"
grafana:
install_sources: "deb http://${APT_MIRROR_HOST}/packagecloud.io/grafana/stable/debian/ stretch main"
elasticsearch:
apt-key-url: "http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch"
apt-repository: "deb http://${APT_MIRROR_HOST}/artifacts.elastic.co/packages/5.x/apt stable main"
filebeat:
logpath: '/var/log/*.log /var/log/containers/*.log'
install_sources: "deb http://${APT_MIRROR_HOST}/artifacts.elastic.co/packages/5.x/apt stable main"
install_keys: $(echo -e "|\n - |\n$(wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch | sed -e 's/^/ /')")
telegraf:
install_sources: "deb http://${APT_MIRROR_HOST}/ppa.launchpad.net/telegraf-devs/ppa/ubuntu xenial main"
install_keys: $(echo -e "|\n - |\n$(wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-telegraf | sed -e 's/^/ /')")
EOL
}
juju_add_relations(){
# add relations
juju relate kubernetes-master:kube-api-endpoint kubeapi-load-balancer:apiserver
juju relate kubernetes-master:loadbalancer kubeapi-load-balancer:loadbalancer
juju relate kubernetes-master:kube-control kubernetes-worker:kube-control
juju relate kubernetes-master:certificates easyrsa:client
juju relate etcd:certificates easyrsa:client
juju relate kubernetes-master:etcd etcd:db
juju relate kubernetes-worker:certificates easyrsa:client
juju relate kubernetes-worker:kube-api-endpoint kubeapi-load-balancer:website
juju relate kubeapi-load-balancer:certificates easyrsa:client
juju relate flannel:etcd etcd:db
juju relate flannel:cni kubernetes-master:cni
juju relate flannel:cni kubernetes-worker:cni
juju relate apache2:reverseproxy graylog:website
juju relate graylog:elasticsearch elasticsearch:client
juju relate graylog:mongodb mongodb:database
juju relate filebeat:beats-host kubernetes-master:juju-info
juju relate filebeat:beats-host kubernetes-worker:juju-info
juju relate filebeat:logstash graylog:beats
juju relate prometheus2:grafana-source grafana:grafana-source
juju relate telegraf:prometheus-client prometheus2:target
juju relate kubernetes-master:juju-info telegraf:juju-info
juju relate kubernetes-worker:juju-info telegraf:juju-info
juju relate ceph-mon:osd ceph-osd:mon
juju relate kubernetes-master:ceph-storage ceph-mon:admin
}
adjust_kube_master(){
# kubernetes-master charm doesn't support to change registry address for cdk-addons
kube_master_machines=$(juju status kubernetes-master --format json | jq -r '.machines|length')
installed_cnt=0
while [ $installed_cnt -ne $kube_master_machines ]
do
sleep 10
installed_cnt=$(juju run --application kubernetes-master "snap list | grep cdk-addons" | grep -c cdk-addons || true)
done
juju run --application kubernetes-master -- snap set cdk-addons registry="$PRIV_REGISTRY_HOST"
}
# main
set_vars
check_prerequisite
juju_bootstrap
juju_add_machines
make_config_yaml
juju_deploy
juju_add_relations
# after deployment
echo "sleep for 3 minutes..."
sleep 180
adjust_kube_master
# following is required when deploying charms that uses snap resources to lxd container, because core.snap is not installed in lxd containers
#juju run --machine $id -- "sudo find /var/lib/lxd/containers/ -path "/var/lib/lxd/containers/juju-*/rootfs/home/ubuntu" -type d -exec cp /home/ubuntu/core.snap {} \;"
#juju run --machine $id -- "sudo find /var/lib/lxd/containers/ -path "/var/lib/lxd/containers/juju-*/rootfs/home/ubuntu" -type d -exec cp /home/ubuntu/core.snap {} \;"
#juju run --application prometheus2 "sudo snap install --dangerous /home/ubuntu/core.snap"
#juju run --application graylog "sudo snap install --dangerous /home/ubuntu/core.snap"
echo "disk devices are still not set in ceph-osd. ex) set devices by,"
echo "juju config ceph-osd osd-devices=\"/dev/sdb /dev/sdc\""
Raw
deploy-offline-cdk.sh
#!/usr/bin/env bash
set -e
set -u
## Variables
set_vars(){
APT_MIRROR_HOST=10.12.1.2
LXDKVM_SSTREAM_HOST=10.12.1.2
JUJU_SSTREAM_HOST=10.12.1.2
BOOTSTRAP_NODE_IP=10.12.1.20
CDK_SHRINKWRAP_PATH=$(ls -dt $HOME/cdk-shrinkwrap/*/ | head -1)
PRIV_REGISTRY_HOST=10.12.1.2:5000
MACHINE_IPS="10.12.1.21 \
10.12.1.22 \
10.12.1.23 \
10.12.1.24 \
10.12.1.25 \
10.12.1.26 \
10.12.1.27 \
10.12.1.28 \
10.12.1.29 \
10.12.1.30 \
10.12.1.31 \
10.12.1.32"
KUBE_VER="1.11/stable"
ETCD_VER="3.2/stable"
SERIES=xenial
}
## Bootstrap juju
juju_bootstrap(){
tee ~/mycloud.yaml > /dev/null << EOL
clouds:
manual:
type: manual
endpoint: $BOOTSTRAP_NODE_IP
EOL
juju add-cloud --replace manual ~/mycloud.yaml
echo "bootstrapping juju..."
ssh-keyscan $BOOTSTRAP_NODE_IP
sleep 2
ssh-keyscan $BOOTSTRAP_NODE_IP > ~/.ssh/known_hosts
juju bootstrap --no-gui \
--bootstrap-series=$SERIES \
--config apt-mirror=http://${APT_MIRROR_HOST}/archive.ubuntu.com/ubuntu/ \
--config agent-stream=release \
--config container-image-metadata-url=https://${LXDKVM_SSTREAM_HOST}/lxdkvm/ \
--config agent-metada-url=${JUJU_SSTREAM_HOST}/juju/ \
--debug manual manual-controller > juju-bootstrap.log 2>&1
}
## add machines
juju_add_machines(){
for machine in $MACHINE_IPS
do
ssh-keyscan $machine
sleep 2
ssh-keyscan $machine >> ~/.ssh/known_hosts
juju add-machine --debug ssh:ubuntu@$machine > juju-add-machine-$machine.log 2>&1
done
## prepare machines for deploy
len=$(wc -w <<< "$MACHINE_IPS")
for id in $(seq 0 $(expr $len - 1))
do
juju scp ${CDK_SHRINKWRAP_PATH}/resources/core.snap $id:
#juju scp ${CDK_SHRINKWRAP_PATH}/resources/lxd.snap $id:
juju run --machine $id "sudo snap install --dangerous /home/ubuntu/core.snap"
#juju run --machine $id "sudo snap install --dangerous /home/ubuntu/lxd.snap"
done
}
# deploy all
juju_deploy(){
# deploy etcd
juju deploy -n 3 --resource etcd=${CDK_SHRINKWRAP_PATH}/resources/etcd/etcd.snap --resource snapshot=${CDK_SHRINKWRAP_PATH}/resources/etcd/snapshot.gz \
--to 0,1,2 ${CDK_SHRINKWRAP_PATH}/charms/etcd --config ~/cdk-config.yaml
# deploy kubernetes-master
juju deploy -n 2 --resource cdk-addons=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/cdk-addons.snap \
--resource kube-apiserver=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-apiserver.snap \
--resource kube-controller-manager=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-controller-manager.snap \
--resource kube-scheduler=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-scheduler.snap \
--resource kubectl=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kubectl.snap --to 3,4 ${CDK_SHRINKWRAP_PATH}/charms/kubernetes-master --config ~/cdk-config.yaml
# deploy flannel
juju deploy --resource flannel-amd64=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-amd64.gz --resource flannel-arm64=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-arm64.gz \
--resource flannel-s390x=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-s390x.gz ${CDK_SHRINKWRAP_PATH}/charms/flannel
# deploy easyrsa
juju deploy -n 1 --resource easyrsa=${CDK_SHRINKWRAP_PATH}/resources/easyrsa/easyrsa.tgz --to 0 ${CDK_SHRINKWRAP_PATH}/charms/easyrsa
# deploy kubernetes-worker
juju deploy -n 5 --resource cni-amd64=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-amd64.tgz --resource cni-arm64=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-arm64.tgz \
--resource cni-s390x=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-s390x.tgz --resource kube-proxy=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kube-proxy.snap \
--resource kubectl=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kubectl.snap --resource kubelet=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kubelet.snap \
--to 5,6,7,8,9 ${CDK_SHRINKWRAP_PATH}/charms/kubernetes-worker --config ~/cdk-config.yaml
# deploy kubeapi-load-balancer
juju deploy -n 1 --to 10 ${CDK_SHRINKWRAP_PATH}/charms/kubeapi-load-balancer
# deploy prometheus
#juju deploy -n 1 --to 10 ${CDK_SHRINKWRAP_PATH}/charms/prometheus
juju deploy -n 1 --resource prometheus=${CDK_SHRINKWRAP_PATH}/resources/prometheus/prometheus.snap --to 10 ${CDK_SHRINKWRAP_PATH}/charms/prometheus2
# deploy grafana
juju deploy -n 1 --to 10 ${CDK_SHRINKWRAP_PATH}/charms/grafana --config ~/cdk-config.yaml
# deploy telegraf
juju deploy ${CDK_SHRINKWRAP_PATH}/charms/telegraf --config ~/cdk-config.yaml
# deploy elasticsearch
juju deploy -n 1 --to 11 ${CDK_SHRINKWRAP_PATH}/charms/elasticsearch --config ~/cdk-config.yaml
# deploy filebeat
juju deploy ${CDK_SHRINKWRAP_PATH}/charms/filebeat --config ~/cdk-config.yaml
# deploy graylog
juju deploy -n 1 --resource graylog=${CDK_SHRINKWRAP_PATH}/resources/graylog/graylog.snap --to 11 ${CDK_SHRINKWRAP_PATH}/charms/graylog
# deploy mongodb
juju deploy -n 1 --to 11 ${CDK_SHRINKWRAP_PATH}/charms/mongodb
# deploy apache2
juju deploy -n 1 --to 11 ${CDK_SHRINKWRAP_PATH}/charms/apache2 --config ~/cdk-config.yaml
# deploy ceph-osd
juju deploy -n 5 --to 5,6,7,8,9 $CDK_SHRINKWRAP_PATH/charms/ceph-osd
# deploy ceph-mon
juju deploy -n 5 --to 5,6,7,8,9 $CDK_SHRINKWRAP_PATH/charms/ceph-mon
}
make_config_yaml(){
tee ~/cdk-config.yaml > /dev/null << EOL
kubernetes-master:
channel: $KUBE_VER
kubernetes-worker:
kubelet-extra-args: "pod-infra-container-image=$PRIV_REGISTRY_HOST/google_containers/pause-amd64:3.1"
docker-opts: "--insecure-registry=$PRIV_REGISTRY_HOST"
nginx-image: "$PRIV_REGISTRY_HOST/nginx-ingress-controller:0.16.1"
default-backend-image: "$PRIV_REGISTRY_HOST/defaultbackend:1.4"
channel: $KUBE_VER
etcd:
channel: $ETCD_VER
apache2:
enable_modules: "headers proxy_html proxy_http"
grafana:
install_sources: "deb http://${APT_MIRROR_HOST}/packagecloud.io/grafana/stable/debian/ stretch main"
elasticsearch:
apt-key-url: "http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch"
apt-repository: "deb http://${APT_MIRROR_HOST}/artifacts.elastic.co/packages/5.x/apt stable main"
filebeat:
logpath: '/var/log/*.log /var/log/containers/*.log'
install_sources: "deb http://${APT_MIRROR_HOST}/artifacts.elastic.co/packages/5.x/apt stable main"
install_keys: $(echo -e "|\n - |\n$(wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch | sed -e 's/^/ /')")
telegraf:
install_sources: "deb http://${APT_MIRROR_HOST}/ppa.launchpad.net/telegraf-devs/ppa/ubuntu xenial main"
install_keys: $(echo -e "|\n - |\n$(wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-telegraf | sed -e 's/^/ /')")
EOL
}
juju_add_relations(){
# add relations
juju relate kubernetes-master:kube-api-endpoint kubeapi-load-balancer:apiserver
juju relate kubernetes-master:loadbalancer kubeapi-load-balancer:loadbalancer
juju relate kubernetes-master:kube-control kubernetes-worker:kube-control
juju relate kubernetes-master:certificates easyrsa:client
juju relate etcd:certificates easyrsa:client
juju relate kubernetes-master:etcd etcd:db
juju relate kubernetes-worker:certificates easyrsa:client
juju relate kubernetes-worker:kube-api-endpoint kubeapi-load-balancer:website
juju relate kubeapi-load-balancer:certificates easyrsa:client
juju relate flannel:etcd etcd:db
juju relate flannel:cni kubernetes-master:cni
juju relate flannel:cni kubernetes-worker:cni
juju relate apache2:reverseproxy graylog:website
juju relate graylog:elasticsearch elasticsearch:client
juju relate graylog:mongodb mongodb:database
juju relate filebeat:beats-host kubernetes-master:juju-info
juju relate filebeat:beats-host kubernetes-worker:juju-info
juju relate filebeat:logstash graylog:beats
#juju relate prometheus:grafana-source grafana:grafana-source
#juju relate telegraf:prometheus-client prometheus:target
juju relate prometheus2:grafana-source grafana:grafana-source
juju relate telegraf:prometheus-client prometheus2:target
juju relate kubernetes-master:juju-info telegraf:juju-info
juju relate kubernetes-worker:juju-info telegraf:juju-info
juju relate ceph-mon:osd ceph-osd:mon
juju relate kubernetes-master:ceph-storage ceph-mon:admin
}
# main
set_vars
juju_bootstrap
juju_add_machines
make_config_yaml
juju_deploy
juju_add_relations
# after deployment
sleep 300
# juju won't replace securiy.ubuntu.com to value in --apt-mirror
juju run --all -- sudo sed -i 's/security.ubuntu.com/10.12.1.2\/archive.ubuntu.com/g' /etc/apt/sources.list
# kubernetes-master charm doesn't support to change registry address for cdk-addons
juju run --application kubernetes-master -- snap set cdk-addons registry="$PRIV_REGISTRY_HOST"
# elasticsearch charm doesn't support to pass GPG key, but only url
#juju run --application elasticsearch -- wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch | sudo apt-key add -
echo "disk devices are still not set in ceph-osd. set devices by,"
echo "juju config ceph-osd osd-devices=\"/dev/sdb /dev/sdc\""
Raw
prepare-offline-cdk.sh
#!/usr/bin/env bash
## Variables
APT_MIRROR_PATH="/var/spool/apt-mirror/"
SSTREAM_PATH="/var/spool/sstreams/"
GPG_KEY_TELEGRAF=C94406F5
GPG_KEY_ELASTICSEARCH=D88E42B4
SERIES=xenial
CERT_PATH="/etc/pki/tls/certs/"
PRIV_KEY_PATH="/etc/pki/tls/private/"
CERT_FILE="mirror.crt"
PRIV_KEY_FILE="mirror.key"
## Install all necessary packages
echo "Installing Missing Packages & Repositories"
sudo apt-add-repository -y ppa:telegraf-devs/ppa
sudo apt-add-repository -y ppa:juju/stable
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
wget -qO - https://packagecloud.io/gpg.key | sudo apt-key add -
sudo apt update && sudo apt install -y apt-mirror docker.io git apache2 python3-pip unzip juju
sudo pip3 install pyyaml
sudo pip3 install pyaml
sudo snap install kubectl --classic
sudo snap install charm
## Setup apt-mirror
echo "Configuring apt-mirror"
sudo tee /etc/apt/mirror.list > /dev/null <<EOL
set base_path $APT_MIRROR_PATH
set nthreads 20
set _tilde 0
deb http://archive.ubuntu.com/ubuntu xenial main restricted universe multiverse
deb http://archive.ubuntu.com/ubuntu xenial-security main restricted universe multiverse
deb http://archive.ubuntu.com/ubuntu xenial-updates main restricted universe multiverse
deb http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu xenial main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu xenial-security main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu xenial-updates main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse
deb http://ppa.launchpad.net/telegraf-devs/ppa/ubuntu xenial main
deb https://artifacts.elastic.co/packages/5.x/apt stable main
deb https://packagecloud.io/grafana/stable/debian/ stretch main
clean http://archive.ubuntu.com/ubuntu
clean http://ppa.launchpad.net/telegraf-devs/ppa/ubuntu
clean https://artifacts.elastic.co/packages/5.x/apt
clean https://packagecloud.io/grafana/stable/debian/
EOL
# save GPG keys
apt_key_path="$APT_MIRROR_PATH"mirror/keys/
sudo mkdir -p $apt_key_path
apt-key export $GPG_KEY_TELEGRAF | sudo tee "$apt_key_path"GPG-KEY-telegraf
apt-key export $GPG_KEY_ELASTICSEARCH | sudo tee "$apt_key_path"GPG-KEY-elasticsearch
echo "Start syncing Files, this will take few hours"
# syncing docker images
unset -e
sudo docker run -d -p 5000:5000 --restart=always --name registry registry:2
export REGISTRY="localhost:5000"
set -e
sudo docker pull gcr.io/google_containers/pause-amd64:3.1
sudo docker tag gcr.io/google_containers/pause-amd64:3.1 "$REGISTRY"/google_containers/pause-amd64:3.1
sudo docker push "$REGISTRY"/google_containers/pause-amd64:3.1
sudo docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.1
sudo docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.1 "$REGISTRY"/nginx-ingress-controller:0.16.1
sudo docker push "$REGISTRY"/nginx-ingress-controller:0.16.1
sudo docker pull k8s.gcr.io/defaultbackend:1.4
sudo docker tag k8s.gcr.io/defaultbackend:1.4 "$REGISTRY"/defaultbackend:1.4
sudo docker push "$REGISTRY"/defaultbackend:1.4
sudo docker pull cdkbot/addon-resizer-amd64:1.8.1
sudo docker tag cdkbot/addon-resizer-amd64:1.8.1 ${REGISTRY}/addon-resizer-amd64:1.8.1
sudo docker push ${REGISTRY}/addon-resizer-amd64:1.8.1
sudo docker pull k8s.gcr.io/heapster-amd64:v1.5.3
sudo docker tag k8s.gcr.io/heapster-amd64:v1.5.3 ${REGISTRY}/heapster-amd64:v1.5.3
sudo docker push ${REGISTRY}/heapster-amd64:v1.5.3
sudo docker pull k8s.gcr.io/heapster-influxdb-amd64:v1.3.3
sudo docker tag k8s.gcr.io/heapster-influxdb-amd64:v1.3.3 ${REGISTRY}/heapster-influxdb-amd64:v1.3.3
sudo docker push ${REGISTRY}/heapster-influxdb-amd64:v1.3.3
sudo docker pull k8s.gcr.io/heapster-grafana-amd64:v4.4.3
sudo docker tag k8s.gcr.io/heapster-grafana-amd64:v4.4.3 ${REGISTRY}/heapster-grafana-amd64:v4.4.3
sudo docker push ${REGISTRY}/heapster-grafana-amd64:v4.4.3
sudo docker pull k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.10
sudo docker tag k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.10 ${REGISTRY}/k8s-dns-kube-dns-amd64:1.14.10
sudo docker push ${REGISTRY}/k8s-dns-kube-dns-amd64:1.14.10
sudo docker pull k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.10
sudo docker tag k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.10 ${REGISTRY}/k8s-dns-dnsmasq-nanny-amd64:1.14.10
sudo docker push ${REGISTRY}/k8s-dns-dnsmasq-nanny-amd64:1.14.10
sudo docker pull k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.10
sudo docker tag k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.10 ${REGISTRY}/k8s-dns-sidecar-amd64:1.14.10
sudo docker push ${REGISTRY}/k8s-dns-sidecar-amd64:1.14.10
sudo docker pull k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3
sudo docker tag k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3 ${REGISTRY}/kubernetes-dashboard-amd64:v1.8.3
sudo docker push ${REGISTRY}/kubernetes-dashboard-amd64:v1.8.3
sudo docker pull k8s.gcr.io/metrics-server-amd64:v0.2.1
sudo docker tag k8s.gcr.io/metrics-server-amd64:v0.2.1 ${REGISTRY}/metrics-server-amd64:v0.2.1
sudo docker push ${REGISTRY}/metrics-server-amd64:v0.2.1
# pull nexus for a private registry (optional)
sudo docker pull sonatype/nexus3:latest
sudo docker tag sonatype/nexus3:latest ${REGISTRY}/nexus3:latest
sudo docker push ${REGISTRY}/nexus3:latest
# pull rancher for management (optional)
sudo docker pull rancher/rancher:latest
sudo docker tag rancher/rancher:latest ${REGISTRY}/rancher:latest
sudo docker push ${REGISTRY}/rancher:latest
# synch apt packages (this will take several hours)
sudo apt-mirror
# synch simplestreams metadata
workdir=${SSTREAM_PATH}juju
sudo sstream-mirror --no-verify --progress --max=1 --path=streams/v1/index2.sjson https://streams.canonical.com/juju/tools/ $workdir 'arch=amd64' 'release~(xenial|bionic)' 'version~(2.2|2.3|2.4)'
workdir=${SSTREAM_PATH}lxdkvm
sudo sstream-mirror --keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg --progress --max=1 --path=streams/v1/index.json https://cloud-images.ubuntu.com/releases/ $workdir/_latest 'arch=amd64' 'release~(trusty|xenial)' 'ftype~(lxd.tar.xz|squashfs|root.tar.xz|root.tar.gz|disk1.img|.json|.sjson)'
# Running CDK Shrink Wrap
if [ -d cdk-shrinkwrap ]; then
cd cdk-shrinkwrap
git pull
else
git clone https://github.com/juju-solutions/cdk-shrinkwrap.git
cd cdk-shrinkwrap
fi
./shrinkwrap.py canonical-kubernetes --channel stable
cdk_shrinkwrap_name=$(ls -t canonical-kubernetes-stable-*.tar.gz | head -1 | cut -d'.' -f1)
tar xf ${cdk_shrinkwrap_name}.tar.gz
cdk_shrinkwrap_path=$HOME/cdk-shrinkwrap/${cdk_shrinkwrap_name}
# Pull missing charms
charms="ceph-osd ceph-mon vault prometheus2 prometheus grafana telegraf elasticsearch filebeat graylog mongodb apache2"
for charm in $charms
do
charm pull $SERIES/$charm ${cdk_shrinkwrap_path}/charms/$charm >> downloaded.txt
done
container_charms="canal calico"
for charm in $container_charms
do
charm pull cs:~containers/$SERIES/$charm ${cdk_shrinkwrap_path}/charms/$charm >> downloaded.txt
done
# Pull missing snaps for graylog and prometheus
snap download graylog --stable
#snap download prometheus --stable
snap download --channel=2/stable prometheus
mkdir -p ${cdk_shrinkwrap_path}/resources/graylog
mkdir -p ${cdk_shrinkwrap_path}/resources/prometheus
rm *.assert
mv graylog* ${cdk_shrinkwrap_path}/resources/graylog/graylog.snap
mv prometheus* ${cdk_shrinkwrap_path}/resources/prometheus/prometheus.snap
# Configure Mirror to Serve Repo to other nodes
sudo tee /etc/apache2/sites-available/sstreams-mirror.conf > /dev/null <<EOL
<VirtualHost *:443>
ServerName sstreams.cdk-juju
ServerAlias *
DocumentRoot ${SSTREAM_PATH}
SSLCACertificatePath /etc/ssl/certs
SSLCertificateFile /etc/pki/tls/certs/mirror.crt
SSLEngine On
SSLCertificateKeyFile /etc/pki/tls/private/mirror.key
LogLevel info
ErrorLog /var/log/apache2/mirror-lxdkvm-error.log
CustomLog /var/log/apache2/mirror-lxdkvm-access.log combined
<Directory ${SSTREAM_PATH}>
Options Indexes FollowSymLinks
AllowOverride None
Require all granted
</Directory>
</VirtualHost>
EOL
sudo tee /etc/apache2/sites-available/ubuntu-mirror.conf > /dev/null <<EOL
<VirtualHost *:80>
ServerName cdk-juju
DocumentRoot ${APT_MIRROR_PATH}mirror/
LogLevel info
ErrorLog /var/log/apache2/mirror-error.log
CustomLog /var/log/apache2/mirror-access.log combined
<Directory $APT_MIRROR_PATH>
Options Indexes FollowSymLinks
AllowOverride None
Require all granted
</Directory>
</VirtualHost>
EOL
# Generate SSL cert to be used by apache
sudo mkdir -p $PRIV_KEY_PATH
sudo mkdir -p $CERT_PATH
PRIMARYIP=`hostname -i`
sudo tee /root/$HOSTNAME.conf > /dev/null <<EOL
[ req ]
prompt = no
default_bits = 4096
distinguished_name = req_distinguished_name
req_extensions = req_ext
[ req_distinguished_name ]
C=GB
ST=London
L=London
O=Canonical
OU=Canonical
CN=$HOSTNAME
[ req_ext ]
subjectAltName = @alt_names
[alt_names]
DNS.1 = $HOSTNAME
DNS.2 = $PRIMARYIP
IP.1 = $PRIMARYIP
EOL
sudo openssl req \
-new \
-newkey rsa:4096 \
-days 3650 \
-nodes \
-x509 \
-config /root/$HOSTNAME.conf \
-keyout ${PRIV_KEY_PATH}${PRIV_KEY_FILE} \
-out ${CERT_PATH}${CERT_FILE}
sudo a2enmod ssl
sudo a2ensite sstreams-mirror.conf
sudo a2ensite ubuntu-mirror.conf
sudo systemctl restart apache2
echo "Repo configuration and sync done, exiting...!"
echo "Ubuntu repo: http://$HOSTNAME/ubuntu/"
echo "LXD and KVM metadata: https://$HOSTNAME/lxdkvm/_latest"
echo "Juju metadata: https://$HOSTNAME/juju/"
echo "You should also be able to use the IP address"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment