ceph encrypted bluestore osd

mds.sh

mkdir -p /var/lib/ceph/mds/ceph-$(hostname -s)/
ceph-authtool --create-keyring /var/lib/ceph/mds/ceph-$(hostname -s)/keyring --gen-key -n mds.$(hostname -s)
ceph auth add mds.$(hostname -s) osd "allow rwx" mds "allow" mon "allow profile mds" -i /var/lib/ceph/mds/ceph-$(hostname -s)/keyring
sudo chown -R ceph:ceph /var/lib/ceph
systemctl enable ceph-mds@$(hostname -s)
systemctl start ceph-mds@$(hostname -s)

osd-encrypted.sh

mkdir -p /etc/luks/

DRIVES=(sdc sdd sde sdf)

for DRIVE in $DRIVES; do
    sgdisk --zap-all /dev/${DRIVE}
    sleep 5
    sgdisk --new 1:0:+128M --typecode 1:FD00 /dev/${DRIVE}
    sgdisk --new 2:0:0     --typecode 1:FD00 /dev/${DRIVE}
    sleep 5
    partprobe
    sleep 5
    OSD_NUM=$(ceph osd create)
    for PART_NUM in 1 2; do
        PART_UUID=$(blkid /dev/${DRIVE}${PART_NUM} -s PARTUUID -o value)
        dd bs=512 count=4 if=/dev/urandom of=/etc/luks/${PART_UUID}.key
        cryptsetup luksFormat --batch-mode --key-file=/etc/luks/${PART_UUID}.key /dev/disk/by-partuuid/${PART_UUID}
        cryptsetup luksOpen --batch-mode --key-file=/etc/luks/${PART_UUID}.key /dev/disk/by-partuuid/${PART_UUID} osd-${OSD_NUM}-${PART_NUM}
        echo "osd-${OSD_NUM}-${PART_NUM} PARTUUID=${PART_UUID} /etc/luks/${PART_UUID}.key luks,timeout=180" >> /etc/crypttab
        if [ $PART_NUM -eq 1 ]; then
            mkfs.xfs /dev/mapper/osd-${OSD_NUM}-${PART_NUM}
            mkdir -p /var/lib/ceph/osd/ceph-${OSD_NUM}
            echo "/dev/mapper/osd-${OSD_NUM}-${PART_NUM} /var/lib/ceph/osd/ceph-${OSD_NUM} xfs defaults 0 2" >> /etc/fstab
        fi
    done
    cat >> /etc/ceph/ceph.conf <<DELIM

[osd.${OSD_NUM}]
host = $(hostname -s)
osd data = /var/lib/ceph/osd/ceph-${OSD_NUM}
bluestore block path = /dev/mapper/osd-${OSD_NUM}-2
DELIM
done

osd-luks.sh

mkdir -p /etc/luks/

DRIVES=(sdd sde)

for DRIVE in $DRIVES; do
    sgdisk --zap-all /dev/${DRIVE}
    sgdisk --new 1:0:+128M --typecode 1:FD00 /dev/${DRIVE}
    sgdisk --new 2:0:+20G  --typecode 1:FD00 /dev/${DRIVE}
    sgdisk --new 3:0:+1G   --typecode 1:FD00 /dev/${DRIVE}
    sgdisk --new 4:0:0     --typecode 1:FD00 /dev/${DRIVE}
    sleep 1
    partprobe
    OSD_NUM=$(ceph osd create)
    for PART_NUM in 1 2 3 4; do
        PART_UUID=$(blkid /dev/${DRIVE}${PART_NUM} -s PARTUUID -o value)
        dd bs=512 count=4 if=/dev/urandom of=/etc/luks/${PART_UUID}.key
        cryptsetup luksFormat --batch-mode --key-file=/etc/luks/${PART_UUID}.key /dev/disk/by-partuuid/${PART_UUID}
        cryptsetup luksOpen --batch-mode --key-file=/etc/luks/${PART_UUID}.key /dev/disk/by-partuuid/${PART_UUID} osd-${OSD_NUM}-${PART_NUM}
        echo "osd-${OSD_NUM}-${PART_NUM} PARTUUID=${PART_UUID} /etc/luks/${PART_UUID}.key luks,timeout=180" >> /etc/crypttab
        if [ $PART_NUM -eq 1 ]; then
            mkfs.xfs /dev/mapper/osd-${OSD_NUM}-${PART_NUM}
            mkdir -p /var/lib/ceph/osd/ceph-${OSD_NUM}
            echo "/dev/mapper/osd-${OSD_NUM}-${PART_NUM} /var/lib/ceph/osd/ceph-${OSD_NUM} xfs defaults 0 2" >> /etc/fstab
        fi
    done
    cat >> /etc/ceph/ceph.conf <<DELIM
    [osd.${OSD_NUM}]
    host = $(hostname -s)
    osd data = /var/lib/ceph/osd/ceph-${OSD_NUM}
    bluestore block path = /dev/mapper/osd-${OSD_NUM}-4
    bluestore block db path = /dev/mapper/osd-${OSD_NUM}-2
    bluestore block wal path = /dev/mapper/osd-${OSD_NUM}-3
    DELIM
done

osd-unecrypted.sh

DRIVES=(sde)

for DRIVE in $DRIVES; do
    sgdisk --zap-all /dev/${DRIVE}
    sgdisk \
        --new 1:0:+128M --typecode 1:8300 \
        --new 2:0:+20G  --typecode 2:fd00 \
        --new 3:0:+1G   --typecode 3:fd00 \
        --new 4:0:0     --typecode 4:fd00 \
        /dev/${DRIVE}
    sleep 5

    # OSD_NUM=13
    OSD_NUM=$(ceph osd create)
    for PART_NUM in 1 2 3 4; do
        PART_UUID=$(blkid /dev/${DRIVE}${PART_NUM} -s PARTUUID -o value)
        if [ $PART_NUM -eq 1 ]; then
            mkfs.xfs -f /dev/disk/by-partuuid/${PART_UUID}
            mkdir -p /var/lib/ceph/osd/ceph-${OSD_NUM}
            echo "PARTUUID=${PART_UUID} /var/lib/ceph/osd/ceph-${OSD_NUM} xfs defaults 0 2" >> /etc/fstab
        fi
    done
    PART_UUID_BLOCK=$(blkid /dev/${DRIVE}4 -s PARTUUID -o value)
    PART_UUID_DB=$(blkid /dev/${DRIVE}2 -s PARTUUID -o value)
    PART_UUID_WAL=$(blkid /dev/${DRIVE}3 -s PARTUUID -o value)
    cat >> /etc/ceph/ceph.conf <<DELIM
[osd.${OSD_NUM}]
host = $(hostname -s)
osd data = /var/lib/ceph/osd/ceph-${OSD_NUM}
osd objectstore = bluestore
bluestore block path = /dev/disk/by-partuuid/${PART_UUID_BLOCK}
bluestore block db path = /dev/disk/by-partuuid/${PART_UUID_DB}
bluestore block wal path = /dev/disk/by-partuuid/${PART_UUID_WAL}
DELIM
#ceph-osd -i ${OSD_NUM} --mkfs --mkkey
#ceph auth add osd.${OSD_NUM} osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-${OSD_NUM}/keyring
#ceph osd crush add osd.${OSD_NUM} 1.0 host=$(hostname -s)
done