Increase replication factor for __consumer_offsets Kafka topic

01_IncreaseReplicationFactor.md

Increasing replication factor for a topic

1. Create a custom reassignment plan (see attached file inc-replication-factor.json). In this case we are going from replication factor of 1 to 3.

2. Run Kafka partition reassignment script:

   kafka-reassign-partitions --zookeeper $ZOOKEEPER_CONNECT \
       --reassignment-json-file /home/liquidnt/inc-replication-factor.json  \
       --execute
   

3. Verify if the assignment was successful

   kafka-reassign-partitions --zookeeper $ZOOKEEPER_CONNECT \
       --reassignment-json-file /home/liquidnt/inc-replication-factor.json  \
       --verify
   

4. More verifications

    ./kafka-topics --zookeeper $ZOOKEEPER_CONNECT --describe --topic __consumer_offsets
   

inc-replication-factor.json

{"version":1,
 "partitions":[
   {"topic":"__consumer_offsets", "partition":0,  "replicas":[0, 1, 2]},
   {"topic":"__consumer_offsets", "partition":1,  "replicas":[0, 1, 2]},
   {"topic":"__consumer_offsets", "partition":2,  "replicas":[0, 1, 2]},
   {"topic":"__consumer_offsets", "partition":3,  "replicas":[0, 1, 2]},
   {"topic":"__consumer_offsets", "partition":4,  "replicas":[0, 1, 2]},
   {"topic":"__consumer_offsets", "partition":5,  "replicas":[0, 1, 2]},
   {"topic":"__consumer_offsets", "partition":6,  "replicas":[0, 1, 2]},
   {"topic":"__consumer_offsets", "partition":7,  "replicas":[0, 1, 2]},
   {"topic":"__consumer_offsets", "partition":8,  "replicas":[0, 1, 2]},
   {"topic":"__consumer_offsets", "partition":9,  "replicas":[0, 1, 2]},
   {"topic":"__consumer_offsets", "partition":10, "replicas":[1, 2, 3]},
   {"topic":"__consumer_offsets", "partition":11, "replicas":[1, 2, 3]},
   {"topic":"__consumer_offsets", "partition":12, "replicas":[1, 2, 3]},
   {"topic":"__consumer_offsets", "partition":13, "replicas":[1, 2, 3]},
   {"topic":"__consumer_offsets", "partition":14, "replicas":[1, 2, 3]},
   {"topic":"__consumer_offsets", "partition":15, "replicas":[1, 2, 3]},
   {"topic":"__consumer_offsets", "partition":16, "replicas":[1, 2, 3]},
   {"topic":"__consumer_offsets", "partition":17, "replicas":[1, 2, 3]},
   {"topic":"__consumer_offsets", "partition":18, "replicas":[1, 2, 3]},
   {"topic":"__consumer_offsets", "partition":19, "replicas":[1, 2, 3]},
   {"topic":"__consumer_offsets", "partition":20, "replicas":[2, 3, 4]},
   {"topic":"__consumer_offsets", "partition":21, "replicas":[2, 3, 4]},
   {"topic":"__consumer_offsets", "partition":22, "replicas":[2, 3, 4]},
   {"topic":"__consumer_offsets", "partition":23, "replicas":[2, 3, 4]},
   {"topic":"__consumer_offsets", "partition":24, "replicas":[2, 3, 4]},
   {"topic":"__consumer_offsets", "partition":25, "replicas":[2, 3, 4]},
   {"topic":"__consumer_offsets", "partition":26, "replicas":[2, 3, 4]},
   {"topic":"__consumer_offsets", "partition":27, "replicas":[2, 3, 4]},
   {"topic":"__consumer_offsets", "partition":28, "replicas":[2, 3, 4]},
   {"topic":"__consumer_offsets", "partition":29, "replicas":[2, 3, 4]},
   {"topic":"__consumer_offsets", "partition":30, "replicas":[3, 4, 0]},
   {"topic":"__consumer_offsets", "partition":31, "replicas":[3, 4, 0]},
   {"topic":"__consumer_offsets", "partition":32, "replicas":[3, 4, 0]},
   {"topic":"__consumer_offsets", "partition":33, "replicas":[3, 4, 0]},
   {"topic":"__consumer_offsets", "partition":34, "replicas":[3, 4, 0]},
   {"topic":"__consumer_offsets", "partition":35, "replicas":[3, 4, 0]},
   {"topic":"__consumer_offsets", "partition":36, "replicas":[3, 4, 0]},
   {"topic":"__consumer_offsets", "partition":37, "replicas":[3, 4, 0]},
   {"topic":"__consumer_offsets", "partition":38, "replicas":[3, 4, 0]},
   {"topic":"__consumer_offsets", "partition":39, "replicas":[3, 4, 0]},
   {"topic":"__consumer_offsets", "partition":40, "replicas":[4, 0, 1]},
   {"topic":"__consumer_offsets", "partition":41, "replicas":[4, 0, 1]},
   {"topic":"__consumer_offsets", "partition":42, "replicas":[4, 0, 1]},
   {"topic":"__consumer_offsets", "partition":43, "replicas":[4, 0, 1]},
   {"topic":"__consumer_offsets", "partition":44, "replicas":[4, 0, 1]},
   {"topic":"__consumer_offsets", "partition":45, "replicas":[4, 0, 1]},
   {"topic":"__consumer_offsets", "partition":46, "replicas":[4, 0, 1]},
   {"topic":"__consumer_offsets", "partition":47, "replicas":[4, 0, 1]},
   {"topic":"__consumer_offsets", "partition":48, "replicas":[4, 0, 1]},
   {"topic":"__consumer_offsets", "partition":49, "replicas":[4, 0, 1]}
 ]
}