Created
April 17, 2017 18:49
-
-
Save dasl-/901b4b986097761c2a451940965b78de to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/percona-server b/percona-server | |
--- a/percona-server | |
+++ b/percona-server | |
@@ -1 +1 @@ | |
-Subproject commit 1c589f930229a55ca31b4c3648458a027f0f8606 | |
+Subproject commit 1c589f930229a55ca31b4c3648458a027f0f8606-dirty | |
diff --git a/run.sh b/run.sh | |
index c272b7b..ad10901 100755 | |
--- a/run.sh | |
+++ b/run.sh | |
@@ -3,6 +3,33 @@ set -x | |
source common.sh | |
+usage() { | |
+ local exit_code=$1 | |
+ echo "usage: $0 [-n]" | |
+ echo " n Run debezium in 'normal' mode (not using twitter patch)." | |
+ exit $exit_code | |
+} | |
+ | |
+is_twitter_patch=true | |
+while getopts ":nh" opt; do | |
+ case $opt in | |
+ n) | |
+ is_twitter_patch=false | |
+ ;; | |
+ h) | |
+ usage 0 | |
+ ;; | |
+ \?) | |
+ echo "Invalid option: -$OPTARG" >&2 | |
+ usage 1 | |
+ ;; | |
+ :) | |
+ echo "Option -$OPTARG requires an argument." >&2 | |
+ usage 1 | |
+ ;; | |
+ esac | |
+done | |
+ | |
# Clean workspace | |
info "Cleaning up workspace..." | |
rm -rf workspace/mysql-data* | |
@@ -99,30 +126,40 @@ done | |
# Make db tables | |
info "Making db tables..." | |
+sql_log_bin=1 | |
+if $is_twitter_patch | |
+ then | |
+ sql_log_bin=0 | |
+fi | |
for d in $all_mysqlds; do | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE DATABASE etsy_shard_001" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; USE etsy_shard_001" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_shard_001.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_shard_001.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_shard_001.test3 (k1 int, k2 int, v1 int null default '0', primary key (k1, k2))" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE DATABASE etsy_shard_001" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; USE etsy_shard_001" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_shard_001.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_shard_001.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_shard_001.test3 (k1 int, k2 int, v1 int null default '0', primary key (k1, k2))" | |
+ | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE DATABASE etsy_aux" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; USE etsy_aux" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_aux.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_aux.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE DATABASE etsy_aux" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; USE etsy_aux" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_aux.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_aux.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE DATABASE etsy_shard_888" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; USE etsy_shard_888" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_shard_888.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_shard_888.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE DATABASE etsy_shard_888" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; USE etsy_shard_888" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_shard_888.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_shard_888.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE DATABASE etsy_index" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; USE etsy_index" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_index.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_index.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE DATABASE etsy_index" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; USE etsy_index" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_index.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_index.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE DATABASE etsy_crap" | |
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_index.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE DATABASE etsy_crap" | |
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_index.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)" | |
+ if [ "$sql_log_bin" -eq "1" ] | |
+ then | |
+ break # no need to loop; the DDL will replicate. | |
+ fi | |
done | |
# Get the mysql server UUIDs | |
@@ -212,6 +249,11 @@ done | |
# Start debezium | |
info "Starting debezium..." | |
+snapshot_mode="never" | |
+if $is_twitter_patch | |
+ then | |
+ snapshot_mode="twitter_patch" | |
+fi | |
curl -f -m10 -v -i -H "Accept:application/json" -H "Content-Type:application/json" localhost:40003/connectors \ | |
-d '{ | |
"name": "test-connector", | |
@@ -227,7 +269,7 @@ curl -f -m10 -v -i -H "Accept:application/json" -H "Content-Type:application/jso | |
"database.history.kafka.topic": "test", | |
"database.whitelist": "etsy_.+", | |
"database.mysql.expect_metadata_events": true, | |
- "snapshot.mode": "twitter_patch", | |
+ "snapshot.mode": "'"$snapshot_mode"'", | |
"gtid.source.includes": "'"${mysqld_uuid[a]}"'", | |
"transforms": "partitioner,sharded_router,non_sharded_router", | |
"transforms.partitioner.type": "com.etsy.EtsyPartitionAssigner", |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment