Last active
July 6, 2021 09:49
-
-
Save yangl/9c39cd0f10af9313dfcb8c6346890cb3 to your computer and use it in GitHub Desktop.
Kafka MirrorMaker 2.0 单向同步不改为topic名称 https://cwiki.apache.org/confluence/display/KAFKA/KIP-382%3A+MirrorMaker+2.0 MM2现在已支持同步消费组offset同步了 http://kafka.apache.org/documentation/#connectconfigs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
* Licensed to the Apache Software Foundation (ASF) under one or more | |
* contributor license agreements. See the NOTICE file distributed with | |
* this work for additional information regarding copyright ownership. | |
* The ASF licenses this file to You under the Apache License, Version 2.0 | |
* (the "License"); you may not use this file except in compliance with | |
* the License. You may obtain a copy of the License at | |
* | |
* http://www.apache.org/licenses/LICENSE-2.0 | |
* | |
* Unless required by applicable law or agreed to in writing, software | |
* distributed under the License is distributed on an "AS IS" BASIS, | |
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
* See the License for the specific language governing permissions and | |
* limitations under the License. | |
*/ | |
package org.apache.kafka.connect.mirror; | |
import java.util.Map; | |
import org.slf4j.Logger; | |
import org.slf4j.LoggerFactory; | |
/** IdentityReplicationPolicy does not rename remote topics. This is useful for migrating | |
* from legacy MM1, or for any use-case involving one-way replication. | |
* | |
* N.B. MirrorMaker is not able to prevent cycles when using this class, so take care that | |
* your replication topology is acyclic. If migrating from MirrorMaker v1, this will likely | |
* already be the case. | |
*/ | |
public class IdentityReplicationPolicy extends DefaultReplicationPolicy { | |
private static final Logger log = LoggerFactory.getLogger(IdentityReplicationPolicy.class); | |
public static final String SOURCE_CLUSTER_ALIAS_CONFIG = "source.cluster.alias"; | |
private String sourceClusterAlias = null; | |
@Override | |
public void configure(Map<String, ?> props) { | |
super.configure(props); | |
if (props.containsKey(SOURCE_CLUSTER_ALIAS_CONFIG)) { | |
sourceClusterAlias = (String) props.get(SOURCE_CLUSTER_ALIAS_CONFIG); | |
log.info("Using source cluster alias `{}`.", sourceClusterAlias); | |
} | |
} | |
/** Unlike DefaultReplicationPolicy, IdentityReplicationPolicy does not include the source | |
* cluster alias in the remote topic name. Instead, topic names are unchanged. | |
* | |
* In the special case of heartbeats, we defer to DefaultReplicationPolicy. | |
*/ | |
@Override | |
public String formatRemoteTopic(String sourceClusterAlias, String topic) { | |
if (looksLikeHeartbeat(topic)) { | |
return super.formatRemoteTopic(sourceClusterAlias, topic); | |
} else { | |
return topic; | |
} | |
} | |
/** Unlike DefaultReplicationPolicy, IdendityReplicationPolicy cannot know the source of | |
* a remote topic based on its name alone. If `source.cluster.alias` is provided, | |
* `topicSource` will return that. | |
* | |
* In the special case of heartbeats, we defer to DefaultReplicationPolicy. | |
*/ | |
@Override | |
public String topicSource(String topic) { | |
if (looksLikeHeartbeat(topic)) { | |
return super.topicSource(topic); | |
} else { | |
return sourceClusterAlias; | |
} | |
} | |
/** Since any topic may be a "remote topic", this just returns `topic`. | |
* | |
* In the special case of heartbeats, we defer to DefaultReplicationPolicy. | |
*/ | |
@Override | |
public String upstreamTopic(String topic) { | |
if (looksLikeHeartbeat(topic)) { | |
return super.upstreamTopic(topic); | |
} else { | |
return topic; | |
} | |
} | |
private boolean looksLikeHeartbeat(String topic) { | |
return topic != null && topic.endsWith(MirrorClientConfig.HEARTBEATS_TOPIC); | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
clusters=source,target | |
source.bootstrap.servers=100.81.129.73:9092,100.81.129.74:9092,100.81.129.75:9092,100.81.129.76:9092,100.81.129.77:9092 | |
#source.security.protocol=SASL_PLAINTEXT | |
#source.sasl.mechanism=SCRAM-SHA-256 | |
#source.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="admin-secret"; | |
target.bootstrap.servers=100.81.129.68:9092,100.81.129.69:9092,100.81.129.70:9092,100.81.129.71:9092,100.81.129.72:9092 | |
#target.security.protocol=SASL_PLAINTEXT | |
#target.sasl.mechanism=SCRAM-SHA-256 | |
#target.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="admin-secret"; | |
source->target.enabled=true | |
source->target.topics=.* | |
source->target.topics.exclude=KAFKA_BROKER_CHECK_NEW, .*[\-\.]internal, .*\.replica, __.* | |
source->target.groups=.* | |
source->target.groups.exclude=mwopr_consumer, console-consumer-.*, connect-.*, __.* | |
refresh.topics.enabled=true | |
refresh.topics.interval.seconds=10 | |
refresh.groups.enabled=true | |
refresh.groups.interval.seconds=10 | |
sync.topic.configs.enabled=true | |
sync.topic.acls.enabled=true | |
sync.group.offsets.enabled=true | |
sync.group.offsets.interval.seconds=10 | |
emit.heartbeats.enabled=true | |
emit.heartbeats.interval.seconds=5 | |
tasks.max=10 | |
replication.factor=3 | |
replication.policy.class=org.apache.kafka.connect.mirror.IdentityReplicationPolicy | |
#source.consumer.auto.offset.reset=latest | |
source.consumer.isolation.level=read_committed | |
source.consumer.partition.assignment.strategy=org.apache.kafka.clients.consumer.StickyAssignor | |
target.producer.compression.type=snappy | |
target.producer.enable.idempotence=true | |
target.producer.max.in.flight.requests.per.connection=5 | |
target.producer.retries=10 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment