Skip to content

Instantly share code, notes, and snippets.

@replay
Last active July 17, 2018 14:14
Show Gist options
  • Save replay/4cb83d5ed7e9eb83c644cdb04119f177 to your computer and use it in GitHub Desktop.
Save replay/4cb83d5ed7e9eb83c644cdb04119f177 to your computer and use it in GitHub Desktop.
replace broker script
#!/bin/bash
set -x
OLD_BROKER_ID=$1
NEW_BROKER_ID=$2
if [ -z "$OLD_BROKER_ID" ]; then
echo "usage: $0 <oldBroker> <newBroker>"
exit 1
fi
if [ -z "$NEW_BROKER_ID" ]; then
echo "usage: $0 <oldBroker> <newBroker>"
exit 1
fi
TEMP=$(mktemp -d)
THROTTLE=${THROTTLE:-41943040} #40MB/s
BROKER_IDS=$(/opt/kafka/bin/zookeeper-shell.sh zookeeper:2181 <<< "ls /brokers/ids" |grep "^\["|tr -d "[] ")
for T in $(/opt/kafka/bin/kafka-topics.sh --zookeeper=zookeeper:2181 --list| tr '\n' ' ' | head --bytes -1); do
echo "Rebalancing TOPIC $T"
echo '{"version": 1, "topics": [{"topic": "'$T'"}]}' > ${TEMP}/$T.json
# generate reassignemt file
/opt/kafka/bin/kafka-reassign-partitions.sh --zookeeper=zookeeper:2181 --topics-to-move-json-file=${TEMP}/$T.json --broker-list=$BROKER_IDS --generate | head -2 |tail -1 | sed -e "s/$OLD_BROKER_ID/$NEW_BROKER_ID/g" > ${TEMP}/${T}-assignment.json
# execute reassignment
/opt/kafka/bin/kafka-reassign-partitions.sh --zookeeper=zookeeper:2181 --reassignment-json-file=${TEMP}/${T}-assignment.json --execute --throttle $THROTTLE
_running="started"
while [ ! -z "$_running" ]; do
_running=$(/opt/kafka/bin/kafka-reassign-partitions.sh --zookeeper=zookeeper:2181 --reassignment-json-file=${TEMP}/${T}-assignment.json --verify | grep ^Reassignment| awk '{print $NF}' |grep -v successfully)
if [ ! -z "$_running" ]; then
count=$(echo $_running|wc -w)
echo "$count partitions outstanding for topic $T"
sleep 10
fi
done
echo "TOPIC $T rebalanced"
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment