rejoining an etcd2 member after a reboot in a pxe booting cluster

etcd-rejoin-member.sh

#!/bin/sh

set -eo pipefail

usage(){
	echo usage: $0 etcdmember newmember
	exit 1
}

checkhost(){
	local host
	local i=0

	host=$1

	while [[ i -le 20 ]]; do
		echo dialing $host ...
		if ssh -o ConnectTimeout=10 $host true; then
			return 0
		fi
		i=$((i+1))
	done

	if [[ $i -ge 20 ]]; then
		echo host $host timed out
		return 1
	fi
}

die(){
	echo $1
	exit 1
}

ETCD=$1
HOST=$2

if [ -z $ETCD -o -z $HOST ]; then
	usage
fi

# wait a bit for the host to come up
checkhost $HOST || die "host seems down, can't reconfigure it"

# get newmember hostname and peer ip
HOSTNAME=$(ssh $HOST uname -n)
PEERADDR=$(ssh $HOST systemctl show -p Environment etcd2.service | sed -e 's/Environment=//' | tr ' ' '\n' | awk -F=  '/ETCD_INITIAL_ADVERTISE_PEER_URLS/ { print $2 }')

# find the id of the newmember
PEERID=$(ssh $ETCD etcdctl member list | grep $HOSTNAME | cut -d : -f 1)

# remove that peer
ssh $ETCD etcdctl member remove $PEERID

# stop etcd
ssh $HOST sudo systemctl stop etcd2

# wipe data dir
ssh $HOST "sudo rm -rf /var/lib/etcd2/*"

# set existing flag
ssh $HOST sudo tee /etc/systemd/system/etcd2.service.d/80-existing.conf 2>&1 >/dev/null <<EOF
[Service]
Environment="ETCD_INITIAL_CLUSTER_STATE=existing"
EOF

ssh $HOST sudo systemctl daemon-reload

# finally add him back
ssh $ETCD etcdctl member add $HOSTNAME $PEERADDR >/dev/null

# start etcd
ssh $HOST sudo systemctl start etcd2