Skip to content

Instantly share code, notes, and snippets.

@crazytaxii
Last active December 22, 2021 05:26
Show Gist options
  • Select an option

  • Save crazytaxii/f3292e86846b6cafdba3778392241403 to your computer and use it in GitHub Desktop.

Select an option

Save crazytaxii/f3292e86846b6cafdba3778392241403 to your computer and use it in GitHub Desktop.
Deploy 3 nodes Ceph cluster for testing.
#!/bin/bash
set -e
CEPH_VER=${CEPH_VER:-"15.2.1"}
OSD_DEV=${OSD_DEV:-/dev/sdb}
ssh_test() {
if [[ $(ssh root@$1 exit) -ne 0 ]]
then
echo "failed to login $1 as root" && exit 1
fi
}
append_hosts() {
sed -i -e /$2/d /etc/hosts
echo "$1 $2" >> /etc/hosts
}
docker() {
yum install -y yum-utils
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum install docker-ce docker-ce-cli containerd.io -y
systemctl start docker && systemctl enable docker
}
deps() {
ceph_ver=$1
yum install -y wget python3 epel-release
rm -f /etc/yum.repos.d/ceph.repo
cat <<EOF > /etc/yum.repos.d/ceph.repo
[Ceph]
name=Ceph basearch
baseurl=https://download.ceph.com/rpm-$ceph_ver/el7/basearch
enabled=1
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.gpg
[Ceph-noarch]
name=Ceph noarch
baseurl=https://download.ceph.com/rpm-$ceph_ver/el7/noarch
enabled=1
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.gpg
[Ceph-source]
name=Ceph SRPMS
baseurl=https://download.ceph.com/rpm-$ceph_ver/el7/SRPMS
enabled=1
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.gpg
EOF
sed -i -e 's/basearch/\$basearch/g' /etc/yum.repos.d/ceph.repo
rpm --import 'https://download.ceph.com/keys/release.asc'
yum makecache
yum install ceph-common -y
rm -rf /usr/local/bin/cephadm
wget -P /usr/local/bin https://raw.githubusercontent.com/ceph/ceph/v$ceph_ver/src/cephadm/cephadm && chmod +x /usr/local/bin/cephadm
}
chrony() {
cat <<EOF > /etc/chrony.conf
server $1 iburst
# Ignor source level
stratumweight 0
# Record the rate at which the system clock gains/losses time.
driftfile /var/lib/chrony/drift
# Allow the system clock to be stepped in the first three updates
# if its offset is larger than 1 second.
makestep 1.0 3
# Enable kernel synchronization of the real-time clock (RTC).
rtcsync
# Enable hardware timestamping on all interfaces that support it.
#hwtimestamp *
# Increase the minimum number of selectable sources required to adjust
# the system clock.
#minsources 2
# Allow NTP client access from local network.
allow 0.0.0.0/0
bindcmdaddress 127.0.0.1
bindcmdaddress ::1
# Serve time even if not synchronized to a time source.
EOF
if [[ $1 == "localhost" ]]
then
echo "local stratum 10" >> /etc/chrony.conf
fi
cat <<EOF >> /etc/chrony.conf
# Specify file containing keys for NTP authentication.
keyfile /etc/chrony.keys
# Specify directory for log files.
logdir /var/log/chrony
# Select which information is logged.
#log measurements statistics tracking
noclientlog
EOF
systemctl restart chronyd && systemctl enable chronyd
}
main() {
# check
if [[ -z $1 ]]
then
echo "missing IP/hostname of node2" && exit 1
fi
if [[ -z $2 ]]
then
echo "missing IP/hostname of node3" && exit
fi
# IP address of host
ip_addr=$(hostname -I | cut -d ' ' -f 1)
# test ssh connection
for host in localhost $1 $2
do
ssh_test $host
done
# stop and disable firewalld
for host in localhost $1 $2
do
ssh root@$host "systemctl stop firewalld && systemctl disable firewalld"
done
# set up chrony docker and install dependencies
chrony localhost
docker
deps $CEPH_VER
for host in $1 $2
do
ssh root@$host << EOF
$(typeset -f chrony)
chrony $ip_addr
$(typeset -f docker)
docker
$(typeset -f deps)
deps $CEPH_VER
EOF
done
host1=$(hostname)
host2=$(ssh root@$1 "hostname")
host3=$(ssh root@$2 "hostname")
for host in $ip_addr $1 $2
do
ssh root@$host << EOF
$(typeset -f append_hosts)
append_hosts $ip_addr $host1
append_hosts $1 $host2
append_hosts $2 $host3
EOF
done
# bootstrap
export CEPHADM_IMAGE=docker.io/ceph/ceph:v$CEPH_VER
cephadm bootstrap --mon-ip $ip_addr
# wait for 5 mins
echo "wait for 5 mins"
sleep 5m
for host in $host2 $host3
do
ssh-copy-id -f -i /etc/ceph/ceph.pub root@$host
ceph orch host add $host
done
# wait for 5 mins
echo "wait for 5 mins again"
sleep 5m
for host in $host1 $host2 $host3
do
ceph orch daemon add osd $host:$OSD_DEV
done
# wait for 5 mins
sleep 5m
echo "mission complete!"
}
main "$@"
@crazytaxii
Copy link
Copy Markdown
Author

crazytaxii commented Dec 22, 2021

Only for CentOS!
Make sure node1 ssh to node2 and node3 with key pair.

Run this on first node(e.g. 10.211.55.123):

$ curl -OsL https://gist.githubusercontent.com/crazytaxii/f3292e86846b6cafdba3778392241403/raw/111cd1fa522b9b5363fe854cd83e20eec70123f6/ceph3nodes.sh && chmod +x ./ceph3nodes.sh
$ ./ceph3nodes.sh 10.211.55.124 10.211.55.125

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment