Skip to content

Instantly share code, notes, and snippets.

@WalBeh

WalBeh/Setup Digital Ocean Droplet as Gateway for a k3s cluster.md

Last active June 7, 2021 15:16
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save WalBeh/e8b1ee522b57ecfc5eeff6841c10e756 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save WalBeh/e8b1ee522b57ecfc5eeff6841c10e756 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
Setup Digital Ocean Droplet as Gateway for a k3s cluster.md

Setup droplet as Gateway

This might be useful to have k8s nodes without having a public ip address, to simulate an internal network (private IP/LAN) setup

Following: https://docs.digitalocean.com/products/networking/vpc/resources/droplet-as-gateway/

on the worker

ssh -o ProxyCommand="ssh -W %h:%p root@142.93.139.118" root@10.136.16.3

ip route change default via 10.136.16.2
route -n

cat  /etc/netplan/50-cloud-init.yaml
# This file is generated from information provided by the datasource.  Changes
# to it will not persist across an instance reboot.  To disable cloud-init's
# network configuration capabilities, write a file
# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following:
# network: {config: disabled}
network:
    version: 2
    ethernets:
        eth0:
            match:
                macaddress: ce:b9:75:f4:c7:2b
            set-name: eth0
        eth1:
            addresses:
            - 10.136.16.3/20
            match:
                macaddress: 26:e0:44:61:8a:ca
            nameservers:
                addresses:
                - 67.207.67.2
                - 67.207.67.3
                search: []
            routes:
            -    to: 0.0.0.0/0
                 via: 10.136.16.2
            set-name: eth1
			

netplan apply -debug
ip route get 8.8.8.8
cat  >> /etc/hosts << EOF
10.136.16.3 w1
10.136.16.5 w2
10.136.16.4 w3
EOF

k8s view by running k3s on it

nodes

k get nodes -o wide
NAME   STATUS   ROLES    AGE     VERSION         INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
w3     Ready    <none>   5m40s   v1.19.10+k3s1   10.136.16.4   <none>        Ubuntu 18.04.5 LTS   4.15.0-143-generic   containerd://1.4.4-k3s1
w2     Ready    <none>   6m31s   v1.19.10+k3s1   10.136.16.5   <none>        Ubuntu 18.04.5 LTS   4.15.0-143-generic   containerd://1.4.4-k3s1
w1     Ready    master   33m     v1.19.10+k3s1   10.136.16.3   <none>        Ubuntu 18.04.5 LTS   4.15.0-143-generic   containerd://1.4.4-k3s1

pods

k get pods -o wide
NAME                                       READY   STATUS    RESTARTS   AGE     IP           NODE   NOMINATED NODE   READINESS GATES
longhorn-ui-5b864949c4-hkqq9               1/1     Running   0          5m47s   10.42.2.3    w3     <none>           <none>
longhorn-manager-9pgpq                     1/1     Running   0          5m47s   10.42.2.2    w3     <none>           <none>
longhorn-manager-nvz8z                     1/1     Running   1          5m47s   10.42.1.2    w2     <none>           <none>
:
:
:
instance-manager-r-566d9853                1/1     Running   0          5m20s   10.42.0.8    w1     <none>           <none>
instance-manager-e-83a0c04e                1/1     Running   0          5m20s   10.42.0.7    w1     <none>           <none>
engine-image-ei-611d1496-72gp4             1/1     Running   0          5m22s   10.42.0.6    w1     <none>           <none>

svc

k get svc -A
NAMESPACE         NAME                TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                  AGE
default           kubernetes          ClusterIP   10.43.0.1       <none>        443/TCP                  35m
kube-system       kube-dns            ClusterIP   10.43.0.10      <none>        53/UDP,53/TCP,9153/TCP   35m
kube-system       metrics-server      ClusterIP   10.43.237.228   <none>        443/TCP                  35m
longhorn-system   longhorn-backend    ClusterIP   10.43.67.75     <none>        9500/TCP                 6m31s
longhorn-system   longhorn-frontend   ClusterIP   10.43.243.93    <none>        80/TCP                   6m31s
longhorn-system   csi-attacher        ClusterIP   10.43.12.26     <none>        12345/TCP                5m57s
longhorn-system   csi-provisioner     ClusterIP   10.43.85.11     <none>        12345/TCP                5m57s
longhorn-system   csi-resizer         ClusterIP   10.43.70.132    <none>        12345/TCP                5m57s
longhorn-system   csi-snapshotter     ClusterIP   10.43.252.49    <none>        12345/TCP                5m56s
n6x8Nw&86-dgfi5WLMv-7g%a

cratedb pods/svc

kubectl get svc
NAME                                                   TYPE           CLUSTER-IP     EXTERNAL-IP   PORT(S)                         AGE
crate-discovery-5fd1dbb1-9bf8-4212-ad68-6f8f156df934   ClusterIP      10.43.45.249   <none>        4300/TCP                        65s
crate-5fd1dbb1-9bf8-4212-ad68-6f8f156df934             LoadBalancer   10.43.95.118   10.136.16.5   4200:31540/TCP,5432:32358/TCP   66s
root@w1:~# kubectl get pods -o wide
NAME                                                              READY   STATUS     RESTARTS   AGE   IP           NODE   NOMINATED NODE   READINESS GATES
svclb-crate-5fd1dbb1-9bf8-4212-ad68-6f8f156df934-jv95d            2/2     Running    1          82s   10.42.2.18   w3     <none>           <none>
svclb-crate-5fd1dbb1-9bf8-4212-ad68-6f8f156df934-kcqdw            2/2     Running    1          82s   10.42.1.18   w2     <none>           <none>
svclb-crate-5fd1dbb1-9bf8-4212-ad68-6f8f156df934-dmlwz            2/2     Running    0          82s   10.42.0.19   w1     <none>           <none>
crate-data-hot-5fd1dbb1-9bf8-4212-ad68-6f8f156df934-1             0/2     Init:0/3   0          81s   <none>       w1     <none>           <none>
crate-data-hot-5fd1dbb1-9bf8-4212-ad68-6f8f156df934-2             0/2     Init:0/3   0          81s   <none>       w2     <none>           <none>
crate-data-hot-5fd1dbb1-9bf8-4212-ad68-6f8f156df934-0             0/2     Init:0/3   0          81s   <none>       w3     <none>           <none>
backup-metrics-5fd1dbb1-9bf8-4212-ad68-6f8f156df934-5988984bzgv   1/1     Running    0          79s   10.42.0.20   w1     <none>           <none>

external view

;; ANSWER SECTION:
hugo.7961f447a9bb44b98ffdf214fe33ebf8.edge.cratedb-dev.net. 300	IN A 10.136.16.3

curl --insecure https://10.136.16.3:4200/
password authentication failed for user "crate"

Longhorn fio quick check

writing to a longhorn volume with 3 replicas

randrw 60/40 4k iodepth=64

Jobs: 16 (f=16): [m(16)][100.0%][r=2610KiB/s,w=1649KiB/s][r=652,w=412 IOPS][eta 00m:00s]

randrw 60/40 with 1 replica

Jobs: 16 (f=16): [m(16)][100.0%][r=1752KiB/s,w=1232KiB/s][r=438,w=308 IOPS][eta 00m:00s]

writing ot the local disk on the droplet

Jobs: 16 (f=16): [m(16)][100.0%][r=30.0MiB/s,w=20.6MiB/s][r=7932,w=5281 IOPS][eta 00m:00s]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment