Setup DM between MySQL 5.7 and TiDB Sandboxes

dm-sandbox-mysql-to-tidb.sh

#!/bin/bash

killall -9 dm-master
killall -9 dm-worker

# Setup DM between sandboxes
MYSQL_PORT="5725"
TIDB_PORT="3000"

echo "Downloading required packages"
cd ~/sandboxes/msb_3_0_0/tmp
wget http://download.pingcap.org/dm-latest-linux-amd64.tar.gz
wget http://download.pingcap.org/tidb-enterprise-tools-latest-linux-amd64.tar.gz

echo "Installing packages under bin/"
tar -xzf dm-latest-linux-amd64.tar.gz
tar -xzf tidb-enterprise-tools-latest-linux-amd64.tar.gz
mkdir -p ../bin
mv dm-latest-linux-amd64/bin/* ../bin/
mv tidb-enterprise-tools-latest-linux-amd64/bin/* ../bin/
rm -rf tidb-enterprise-tools*
rm -rf dm-latest*
cd ..


cat << EOF > dm-master.toml

# Master Configuration.

#log configuration
log-level = "info"
log-file = "data/dm-master.log"

#dm-master listen address
master-addr = ":8261"

# replication group <-> dm-Worker deployment, we'll refine it when new deployment function is available
[[deploy]]
source-id = "mysql-replica-01"
dm-worker = ":8262"

EOF


cat << EOF >  dm-worker.toml

# Worker Configuration.

#log configuration
log-level = "info"
log-file = "data/dm-worker.log"

#dm-worker listen address
worker-addr = ":8262"

#server id of slave for binlog replication
#each instance (master and slave) in replication group should have different server id
server-id = 101

#represents a MySQL/MariaDB instance or a replication group
source-id = "mysql-replica-01"

#flavor: mysql/mariadb
flavor = "mysql"

#directory that used to store relay log
relay-dir = "./relay_log"

#enable gtid in relay log unit
enable-gtid = false

#charset of DSN of source mysql/mariadb instance
# charset= ""

[from]
host = "127.0.0.1"
user = "msandbox"
password = "UakOD6Hj2HezXTrta6Qs40DFFGoDWMkcgQ=="
port = $MYSQL_PORT

EOF

cat << EOF > task.yaml

---
name: test
task-mode: all
is-sharding: false
meta-schema: "dm_meta"
remove-meta: false
enable-heartbeat: false

target-database:
  host: "127.0.0.1"
  port: $TIDB_PORT
  user: "root"
  password: "UakOD6Hj2HezXTrta6Qs40DFFGoDWMkcgQ=="

mysql-instances:
  -
    source-id: "mysql-replica-01"

mydumpers:
  global:
    mydumper-path: "./bin/mydumper"
    threads: 4
    chunk-filesize: 64
    skip-tz-utc: true

loaders:
  global:
    pool-size: 16
    dir: "./dumped_data"

syncers:
  global:
    worker-count: 16
    batch: 100
    max-retry: 100

EOF

./bin/dm-master -config dm-master.toml &
./bin/dm-worker -config dm-worker.toml &
sleep 3
echo "start-task task.yaml" | ./bin/dmctl -master-addr 127.0.0.1:8261