Created
April 29, 2022 14:54
-
-
Save jim-cooley/0f7b590c007306930aebe944ce91c13d to your computer and use it in GitHub Desktop.
script to repartition topic-store and other internal topics
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- | |
apiVersion: batch/v1 | |
kind: Job | |
metadata: | |
annotations: | |
strategy.spinnaker.io/recreate: "true" | |
labels: | |
app: repartition-kafka-topic-store | |
team: dbre | |
name: repartition-kafka-topic-store | |
namespace: kafka | |
spec: | |
template: | |
metadata: | |
annotations: | |
strategy.spinnaker.io/recreate: "true" | |
labels: | |
app: repartition-kafka-topic-store | |
team: dbre | |
spec: | |
automountServiceAccountToken: false | |
dnsPolicy: ClusterFirst | |
containers: | |
- image: quay.io/strimzi/kafka:0.27.1-kafka-3.0.0 | |
name: repartition-kafka-topic-store | |
args: | |
- > | |
for repartition_script in $(ls -1A /tmp/config/*.json); do \ | |
bin/kafka-reassign-partitions.sh --bootstrap-server kafka-cluster-kafka-0.kafka-cluster-kafka-brokers.kafka.svc.cluster.local:9092 \ | |
--reassignment-json-file $repartition_script \ | |
--throttle 10000000 \ | |
--execute; | |
sleep 10s; | |
bin/kafka-reassign-partitions.sh --bootstrap-server kafka-cluster-kafka-0.kafka-cluster-kafka-brokers.kafka.svc.cluster.local:9092 \ | |
--reassignment-json-file $repartition_script \ | |
--verify; \ | |
done; sleep 5m; | |
command: [ "/bin/sh", "-c", "-x" ] | |
securityContext: | |
allowPrivilegeEscalation: false | |
privileged: false | |
readOnlyRootFilesystem: false | |
volumeMounts: | |
- name: config | |
mountPath: "/tmp/config" | |
readOnly: true | |
restartPolicy: Never | |
volumes: | |
- name: config | |
configMap: | |
name: repartition-kafka-config | |
--- | |
apiVersion: v1 | |
kind: ConfigMap | |
metadata: | |
annotations: | |
strategy.spinnaker.io/versioned: "false" | |
strategy.spinnaker.io/recreate: "true" | |
name: repartition-kafka-config | |
namespace: kafka | |
data: | |
topic-store.json: | | |
{ | |
"version":1, | |
"partitions":[ | |
{"topic":"__strimzi_store_topic", "partition":0, "replicas": [0,1,2], "log_dirs": ["any","any","any"]}, | |
{"topic":"__strimzi-topic-operator-kstreams-topic-store-changelog", "partition":0, "replicas": [1,0,2], "log_dirs": ["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":0,"replicas":[0,2,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":1,"replicas":[1,0,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":2,"replicas":[2,1,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":3,"replicas":[0,1,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":4,"replicas":[1,2,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":5,"replicas":[2,0,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":6,"replicas":[0,2,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":7,"replicas":[1,0,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":8,"replicas":[2,1,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":9,"replicas":[0,1,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":10,"replicas":[1,2,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":11,"replicas":[2,0,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":12,"replicas":[0,2,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":13,"replicas":[1,0,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":14,"replicas":[2,1,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":15,"replicas":[0,1,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":16,"replicas":[1,2,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":17,"replicas":[2,0,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":18,"replicas":[0,2,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":19,"replicas":[1,0,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":20,"replicas":[2,1,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":21,"replicas":[0,1,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":22,"replicas":[1,2,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":23,"replicas":[2,0,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":24,"replicas":[0,2,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":25,"replicas":[1,0,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":26,"replicas":[2,1,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":27,"replicas":[0,1,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":28,"replicas":[1,2,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":29,"replicas":[2,0,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":30,"replicas":[0,2,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":31,"replicas":[1,0,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":32,"replicas":[2,1,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":33,"replicas":[0,1,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":34,"replicas":[1,2,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":35,"replicas":[2,0,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":36,"replicas":[0,2,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":37,"replicas":[1,0,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":38,"replicas":[2,1,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":39,"replicas":[0,1,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":40,"replicas":[1,2,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":41,"replicas":[2,0,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":42,"replicas":[0,2,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":43,"replicas":[1,0,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":44,"replicas":[2,1,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":45,"replicas":[0,1,2],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":46,"replicas":[1,2,0],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":47,"replicas":[2,0,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":48,"replicas":[0,2,1],"log_dirs":["any","any","any"]}, | |
{"topic":"__consumer_offsets","partition":49,"replicas":[1,0,2],"log_dirs":["any","any","any"]} | |
] | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment