Skip to content

Instantly share code, notes, and snippets.

@dasl-
Created April 17, 2017 18:49
Show Gist options
  • Save dasl-/901b4b986097761c2a451940965b78de to your computer and use it in GitHub Desktop.
Save dasl-/901b4b986097761c2a451940965b78de to your computer and use it in GitHub Desktop.
diff --git a/percona-server b/percona-server
--- a/percona-server
+++ b/percona-server
@@ -1 +1 @@
-Subproject commit 1c589f930229a55ca31b4c3648458a027f0f8606
+Subproject commit 1c589f930229a55ca31b4c3648458a027f0f8606-dirty
diff --git a/run.sh b/run.sh
index c272b7b..ad10901 100755
--- a/run.sh
+++ b/run.sh
@@ -3,6 +3,33 @@ set -x
source common.sh
+usage() {
+ local exit_code=$1
+ echo "usage: $0 [-n]"
+ echo " n Run debezium in 'normal' mode (not using twitter patch)."
+ exit $exit_code
+}
+
+is_twitter_patch=true
+while getopts ":nh" opt; do
+ case $opt in
+ n)
+ is_twitter_patch=false
+ ;;
+ h)
+ usage 0
+ ;;
+ \?)
+ echo "Invalid option: -$OPTARG" >&2
+ usage 1
+ ;;
+ :)
+ echo "Option -$OPTARG requires an argument." >&2
+ usage 1
+ ;;
+ esac
+done
+
# Clean workspace
info "Cleaning up workspace..."
rm -rf workspace/mysql-data*
@@ -99,30 +126,40 @@ done
# Make db tables
info "Making db tables..."
+sql_log_bin=1
+if $is_twitter_patch
+ then
+ sql_log_bin=0
+fi
for d in $all_mysqlds; do
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE DATABASE etsy_shard_001"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; USE etsy_shard_001"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_shard_001.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_shard_001.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_shard_001.test3 (k1 int, k2 int, v1 int null default '0', primary key (k1, k2))"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE DATABASE etsy_shard_001"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; USE etsy_shard_001"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_shard_001.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_shard_001.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_shard_001.test3 (k1 int, k2 int, v1 int null default '0', primary key (k1, k2))"
+
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE DATABASE etsy_aux"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; USE etsy_aux"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_aux.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_aux.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE DATABASE etsy_aux"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; USE etsy_aux"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_aux.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_aux.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE DATABASE etsy_shard_888"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; USE etsy_shard_888"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_shard_888.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_shard_888.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE DATABASE etsy_shard_888"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; USE etsy_shard_888"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_shard_888.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_shard_888.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE DATABASE etsy_index"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; USE etsy_index"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_index.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_index.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE DATABASE etsy_index"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; USE etsy_index"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_index.test (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_index.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE DATABASE etsy_crap"
+ ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=$sql_log_bin; CREATE TABLE etsy_index.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE DATABASE etsy_crap"
- ./workspace/mysql/bin/mysql -uroot -proot --protocol=tcp -P${listen_port[$d]} -e "set sql_log_bin=0; CREATE TABLE etsy_index.test2 (k int primary key, v1 tinyint default '1', v2 int null, v3 int null default '3', v4 int not null default '4', v5 int not null)"
+ if [ "$sql_log_bin" -eq "1" ]
+ then
+ break # no need to loop; the DDL will replicate.
+ fi
done
# Get the mysql server UUIDs
@@ -212,6 +249,11 @@ done
# Start debezium
info "Starting debezium..."
+snapshot_mode="never"
+if $is_twitter_patch
+ then
+ snapshot_mode="twitter_patch"
+fi
curl -f -m10 -v -i -H "Accept:application/json" -H "Content-Type:application/json" localhost:40003/connectors \
-d '{
"name": "test-connector",
@@ -227,7 +269,7 @@ curl -f -m10 -v -i -H "Accept:application/json" -H "Content-Type:application/jso
"database.history.kafka.topic": "test",
"database.whitelist": "etsy_.+",
"database.mysql.expect_metadata_events": true,
- "snapshot.mode": "twitter_patch",
+ "snapshot.mode": "'"$snapshot_mode"'",
"gtid.source.includes": "'"${mysqld_uuid[a]}"'",
"transforms": "partitioner,sharded_router,non_sharded_router",
"transforms.partitioner.type": "com.etsy.EtsyPartitionAssigner",
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment