ceph.sh

#!/bin/bash

set -ex

apt-get update

apt-get install -y \
        apt-transport-https \
        ca-certificates \
        curl \
        gnupg-agent \
        software-properties-common

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -

add-apt-repository \
    "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
    $(lsb_release -cs) \
    stable"

apt-get install -y docker-ce docker-ce-cli containerd.io

CEPH_IMAGE=ceph/daemon:latest-nautilus
HOSTNAME=$(hostname)
PUBLIC_IP=$(ip -4 a show dev eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -1)

mkdir -p \
      /etc/ceph \
      /var/lib/ceph \
      /var/log/ceph

apt-get install -y jq

function wait_status {
    local i=1
    while [ "$i" -le "$2" ]; do
        if docker exec mon ceph -s --format json | jq -r .health.status | grep -q "$1"; then
            return 0
        fi
        ((i=i+1))
        sleep 1
    done
    return 1
}

function wait_rgw {
    local i=1
    while [ "$i" -le "$1" ]; do
        if curl http://localhost:7480; then
            return 0
        fi
        ((i=i+1))
        sleep 1
    done
    return 1
}

docker run -d --restart always --net=host --name=mon \
       -v /etc/ceph:/etc/ceph \
       -v /var/lib/ceph:/var/lib/ceph:rshared \
       -v /var/log/ceph:/var/log/ceph \
       -e MON_IP="$PUBLIC_IP" \
       -e CEPH_PUBLIC_NETWORK=0.0.0.0/0 \
       "$CEPH_IMAGE" mon

wait_status HEALTH_OK 60

cat <<-'EOF' | tee -a /etc/ceph/ceph.conf
osd objectstore = bluestore
osd pool default size = 1

mon allow pool delete = true

log_file = /var/log/ceph/$cluster-$name.log
EOF

docker restart mon

wait_status HEALTH_OK 60

docker run -d --restart always --net=host --name=mgr \
       -v /etc/ceph:/etc/ceph \
       -v /var/lib/ceph:/var/lib/ceph:rshared \
       -v /var/log/ceph:/var/log/ceph \
       "$CEPH_IMAGE" mgr

wait_status HEALTH_WARN 60

mkdir -p /var/local/ceph-disk
dd if=/dev/zero of=/var/local/ceph-disk/0 bs=1G count=10
#losetup --show -Pf /var/local/ceph-disk/0
#mkfs.xfs -f /dev/loop0
#mount /dev/loop0 /var/lib/ceph/osd/

cat <<-'EOF' | tee /etc/systemd/system/losetup.service
[Unit]
Description=Setup loop devices
DefaultDependencies=no
Conflicts=umount.target
Before=local-fs.target
After=systemd-udevd.service
Requires=systemd-udevd.service

[Service]
Type=oneshot
ExecStart=/sbin/losetup /dev/loop0 /var/local/ceph-disk/0
ExecStop=/sbin/losetup -d /dev/loop0
TimeoutSec=60
RemainAfterExit=yes

[Install]
WantedBy=local-fs.target
EOF

systemctl enable losetup.service
systemctl start losetup.service

docker exec mon \
       ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring

docker run -d --restart always --net=host --name=osd \
       -v /etc/ceph:/etc/ceph \
       -v /var/lib/ceph:/var/lib/ceph:rshared \
       -v /var/log/ceph:/var/log/ceph \
       "$CEPH_IMAGE" osd_directory

wait_status HEALTH_OK 60

cat <<-EOF | tee -a /etc/ceph/ceph.conf

[client.rgw.$HOSTNAME]
rgw enable usage log = true
rgw usage log tick interval = 1
rgw usage log flush threshold = 1
rgw usage max shards = 32
rgw usage max user shards = 1
rgw frontends = civetweb port=0.0.0.0:7480
EOF

docker exec mon \
       ceph auth get client.bootstrap-rgw -o /var/lib/ceph/bootstrap-rgw/ceph.keyring

docker run -d --restart always --net=host --name=rgw \
       -v /etc/ceph:/etc/ceph \
       -v /var/lib/ceph:/var/lib/ceph:rshared \
       -v /var/log/ceph:/var/log/ceph \
       "$CEPH_IMAGE" rgw

wait_rgw 120

docker exec rgw radosgw-admin realm create --rgw-realm=us --default
docker exec rgw radosgw-admin zonegroup create --rgw-zonegroup=us-zg01 --endpoints=http://0.0.0.0:7480 --rgw-realm=us --master --default
docker exec rgw radosgw-admin zone create --rgw-zonegroup=us-zg01 --rgw-zone=us-zg01a --master --default --endpoints=http://0.0.0.0:7480
docker exec rgw radosgw-admin period update --commit
docker exec rgw radosgw-admin zonegroup remove --rgw-zonegroup=default --rgw-zone=default
docker exec rgw radosgw-admin period update --commit
docker exec rgw radosgw-admin zone delete --rgw-zone=default
docker exec rgw radosgw-admin period update --commit
docker exec rgw radosgw-admin zonegroup delete --rgw-zonegroup=default
docker exec rgw radosgw-admin period update --commit
docker exec rgw radosgw-admin zonegroup list

echo 'rgw_zone = us-zg01a' >> /etc/ceph/ceph.conf
docker restart rgw
wait_rgw 120

for p in default.rgw.control default.rgw.meta default.rgw.log; do
    docker exec mon ceph osd pool delete "$p" "$p" --yes-i-really-really-mean-it
done

docker exec mon ceph osd pool ls

docker exec rgw \
       radosgw-admin user create \
       --uid=test-admin \
       --display-name=test-admin \
       --access-key=test-admin-access-key \
       --secret-key=test-admin-secret-key

apt-get install -y s3cmd

cat <<-EOF | tee /root/.s3cfg
[default]
access_key = test-admin-access-key
secret_key = test-admin-secret-key
check_ssl_certificate = True
guess_mime_type = True
host_base = 127.0.0.1:7480
host_bucket = 127.0.0.1:7480/%(bucket)
use_https = False
EOF