Skip to content

Instantly share code, notes, and snippets.

@yangl
Last active July 6, 2021 09:49
Show Gist options
  • Save yangl/9c39cd0f10af9313dfcb8c6346890cb3 to your computer and use it in GitHub Desktop.
Save yangl/9c39cd0f10af9313dfcb8c6346890cb3 to your computer and use it in GitHub Desktop.
Kafka MirrorMaker 2.0 单向同步不改为topic名称 https://cwiki.apache.org/confluence/display/KAFKA/KIP-382%3A+MirrorMaker+2.0 MM2现在已支持同步消费组offset同步了 http://kafka.apache.org/documentation/#connectconfigs
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.connect.mirror;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** IdentityReplicationPolicy does not rename remote topics. This is useful for migrating
* from legacy MM1, or for any use-case involving one-way replication.
*
* N.B. MirrorMaker is not able to prevent cycles when using this class, so take care that
* your replication topology is acyclic. If migrating from MirrorMaker v1, this will likely
* already be the case.
*/
public class IdentityReplicationPolicy extends DefaultReplicationPolicy {
private static final Logger log = LoggerFactory.getLogger(IdentityReplicationPolicy.class);
public static final String SOURCE_CLUSTER_ALIAS_CONFIG = "source.cluster.alias";
private String sourceClusterAlias = null;
@Override
public void configure(Map<String, ?> props) {
super.configure(props);
if (props.containsKey(SOURCE_CLUSTER_ALIAS_CONFIG)) {
sourceClusterAlias = (String) props.get(SOURCE_CLUSTER_ALIAS_CONFIG);
log.info("Using source cluster alias `{}`.", sourceClusterAlias);
}
}
/** Unlike DefaultReplicationPolicy, IdentityReplicationPolicy does not include the source
* cluster alias in the remote topic name. Instead, topic names are unchanged.
*
* In the special case of heartbeats, we defer to DefaultReplicationPolicy.
*/
@Override
public String formatRemoteTopic(String sourceClusterAlias, String topic) {
if (looksLikeHeartbeat(topic)) {
return super.formatRemoteTopic(sourceClusterAlias, topic);
} else {
return topic;
}
}
/** Unlike DefaultReplicationPolicy, IdendityReplicationPolicy cannot know the source of
* a remote topic based on its name alone. If `source.cluster.alias` is provided,
* `topicSource` will return that.
*
* In the special case of heartbeats, we defer to DefaultReplicationPolicy.
*/
@Override
public String topicSource(String topic) {
if (looksLikeHeartbeat(topic)) {
return super.topicSource(topic);
} else {
return sourceClusterAlias;
}
}
/** Since any topic may be a "remote topic", this just returns `topic`.
*
* In the special case of heartbeats, we defer to DefaultReplicationPolicy.
*/
@Override
public String upstreamTopic(String topic) {
if (looksLikeHeartbeat(topic)) {
return super.upstreamTopic(topic);
} else {
return topic;
}
}
private boolean looksLikeHeartbeat(String topic) {
return topic != null && topic.endsWith(MirrorClientConfig.HEARTBEATS_TOPIC);
}
}
clusters=source,target
source.bootstrap.servers=100.81.129.73:9092,100.81.129.74:9092,100.81.129.75:9092,100.81.129.76:9092,100.81.129.77:9092
#source.security.protocol=SASL_PLAINTEXT
#source.sasl.mechanism=SCRAM-SHA-256
#source.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="admin-secret";
target.bootstrap.servers=100.81.129.68:9092,100.81.129.69:9092,100.81.129.70:9092,100.81.129.71:9092,100.81.129.72:9092
#target.security.protocol=SASL_PLAINTEXT
#target.sasl.mechanism=SCRAM-SHA-256
#target.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="admin-secret";
source->target.enabled=true
source->target.topics=.*
source->target.topics.exclude=KAFKA_BROKER_CHECK_NEW, .*[\-\.]internal, .*\.replica, __.*
source->target.groups=.*
source->target.groups.exclude=mwopr_consumer, console-consumer-.*, connect-.*, __.*
refresh.topics.enabled=true
refresh.topics.interval.seconds=10
refresh.groups.enabled=true
refresh.groups.interval.seconds=10
sync.topic.configs.enabled=true
sync.topic.acls.enabled=true
sync.group.offsets.enabled=true
sync.group.offsets.interval.seconds=10
emit.heartbeats.enabled=true
emit.heartbeats.interval.seconds=5
tasks.max=10
replication.factor=3
replication.policy.class=org.apache.kafka.connect.mirror.IdentityReplicationPolicy
#source.consumer.auto.offset.reset=latest
source.consumer.isolation.level=read_committed
source.consumer.partition.assignment.strategy=org.apache.kafka.clients.consumer.StickyAssignor
target.producer.compression.type=snappy
target.producer.enable.idempotence=true
target.producer.max.in.flight.requests.per.connection=5
target.producer.retries=10
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment