rook setup on hetzner k3s

00-hetzner-k3s-rook.txt

# one master 3 workers
# each worker has an additional volume attached but not formatted
# https://rook.io/docs/rook/v1.6/ceph-quickstart.html#deploy-the-rook-operator
# https://itnext.io/deploy-a-ceph-cluster-on-kubernetes-with-rook-d75a20c3f5b1
k3sup.exe install --host m1 --user root --ssh-key .\.ssh\id_ed25519 --local-path .kube/config.hetzner --context hetzner
k3sup join --host w2 --server-host m1 --user root --ssh-key .\.ssh\id_ed25519
k3sup join --host w3 --server-host m1 --user root --ssh-key .\.ssh\id_ed25519
k3sup join --host w4 --server-host m1 --user root --ssh-key .\.ssh\id_ed25519
git clone https://github.com/rook/rook.git
cd rook
git branch -r
git checkout release-1.6
cd .\cluster\examples\kubernetes\ceph\
# MODIFY operator.yaml SET 
# ROOK_ENABLE_DISCOVERY_DAEMON: "true"
kubectl create -f crds.yaml -f common.yaml -f operator.yaml
cd -
# wait for resources
# deploy toolbox
kubectl create -f toolbox.yml
# use toolbox
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- bash

[root@rook-ceph-tools-6f94f7d446-5tnpw /]# ceph status
  cluster:
    id:     2d4bbe37-75cc-4f49-b207-b63fa85ef3b7
    health: HEALTH_WARN
            mons are allowing insecure global_id reclaim

  services:
    mon: 3 daemons, quorum a,b,c (age 112s)
    mgr: a(active, since 18s)
    osd: 3 osds: 3 up (since 69s), 3 in (since 69s)

  data:
    pools:   1 pools, 1 pgs
    objects: 3 objects, 0 B
    usage:   3.0 GiB used, 27 GiB / 30 GiB avail
    pgs:     1 active+clean

[root@rook-ceph-tools-6f94f7d446-5tnpw /]# ceph osd df
ID  CLASS  WEIGHT   REWEIGHT  SIZE    RAW USE  DATA     OMAP  META   AVAIL    %USE   VAR   PGS  STATUS
 1    hdd  0.00980   1.00000  10 GiB  1.0 GiB  2.9 MiB   0 B  1 GiB  9.0 GiB  10.03  1.00    1      up
 2    hdd  0.00980   1.00000  10 GiB  1.0 GiB  2.9 MiB   0 B  1 GiB  9.0 GiB  10.03  1.00    1      up
 0    hdd  0.00980   1.00000  10 GiB  1.0 GiB  2.9 MiB   0 B  1 GiB  9.0 GiB  10.03  1.00    1      up
                       TOTAL  30 GiB  3.0 GiB  8.6 MiB   0 B  3 GiB   27 GiB  10.03

# check ceph cluster
PS C:\Users\ferdi\rook> kubectl get cephcluster -A
NAMESPACE   NAME        DATADIRHOSTPATH   MONCOUNT   AGE   PHASE   MESSAGE                        HEALTH        EXTERNAL
rook-ceph   rook-ceph   /var/lib/rook     3          17m   Ready   Cluster created successfully   HEALTH_WARN

# create storageclass
cd -
PS C:\Users\ferdi\rook\rook\cluster\examples\kubernetes\ceph> kubectl apply -f .\csi\rbd\storageclass.yaml
cephblockpool.ceph.rook.io/replicapool created
storageclass.storage.k8s.io/rook-ceph-block created
PS C:\Users\ferdi\rook\rook\cluster\examples\kubernetes\ceph> kubectl get sc
NAME                   PROVISIONER                  RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
local-path (default)   rancher.io/local-path        Delete          WaitForFirstConsumer   false                  39m
rook-ceph-block        rook-ceph.rbd.csi.ceph.com   Delete          Immediate              true                   6s

# use storage class
kubectl create -f pvc.yml

PS C:\Users\ferdi\rook\rook> kubectl get pvc
NAME        STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      AGE
mongo-pvc   Bound    pvc-7c3fa58f-8072-4c38-9c15-8f28ee52897b   5Gi        RWO            rook-ceph-block   76s
PS C:\Users\ferdi\rook\rook> kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM               STORAGECLASS      REASON   AGE
pvc-7c3fa58f-8072-4c38-9c15-8f28ee52897b   5Gi        RWO            Delete           Bound    default/mongo-pvc   rook-ceph-block            76s
PS C:\Users\ferdi\rook\rook>


[root@rook-ceph-tools-6f94f7d446-5tnpw /]# ceph status
  cluster:
    id:     2d4bbe37-75cc-4f49-b207-b63fa85ef3b7
    health: HEALTH_WARN
            mons are allowing insecure global_id reclaim

  services:
    mon: 3 daemons, quorum a,b,c (age 31m)
    mgr: a(active, since 29m)
    osd: 3 osds: 3 up (since 30m), 3 in (since 30m)

  data:
    pools:   2 pools, 33 pgs
    objects: 9 objects, 34 B
    usage:   3.0 GiB used, 27 GiB / 30 GiB avail
    pgs:     33 active+clean

[root@rook-ceph-tools-6f94f7d446-5tnpw /]# ceph config set mon auth_allow_insecure_global_id_reclaim false
[root@rook-ceph-tools-6f94f7d446-5tnpw /]# ceph status
# ... is stuck

pvc.yml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mongo-pvc
spec:
  storageClassName: rook-ceph-block
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 5Gi

toolbox.yml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: rook-ceph-tools
  namespace: rook-ceph
  labels:
    app: rook-ceph-tools
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rook-ceph-tools
  template:
    metadata:
      labels:
        app: rook-ceph-tools
    spec:
      dnsPolicy: ClusterFirstWithHostNet
      containers:
      - name: rook-ceph-tools
        image: rook/ceph:v1.5.10
        command: ["/tini"]
        args: ["-g", "--", "/usr/local/bin/toolbox.sh"]
        imagePullPolicy: IfNotPresent
        env:
          - name: ROOK_CEPH_USERNAME
            valueFrom:
              secretKeyRef:
                name: rook-ceph-mon
                key: ceph-username
          - name: ROOK_CEPH_SECRET
            valueFrom:
              secretKeyRef:
                name: rook-ceph-mon
                key: ceph-secret
        volumeMounts:
          - mountPath: /etc/ceph
            name: ceph-config
          - name: mon-endpoint-volume
            mountPath: /etc/rook
      volumes:
        - name: mon-endpoint-volume
          configMap:
            name: rook-ceph-mon-endpoints
            items:
            - key: data
              path: mon-endpoints
        - name: ceph-config
          emptyDir: {}
      tolerations:
        - key: "node.kubernetes.io/unreachable"
          operator: "Exists"
          effect: "NoExecute"
          tolerationSeconds: 5

What Hetzner specific settings are done? I see only:
#ROOK_ENABLE_DISCOVERY_DAEMON: "true"