[SplitBrain] #docker # docker-compose #failure

example.md

Simulate split brain failure on docker-compose using linux routes

• splitbrain

node splitbrain.js add [2,20] [10,3] true

• disable splitbrain

node splitbrain.js delete [2,20] [10,3]

                        +
                        |
                        |
                        |
                        |       +----------------+
+--------------+        |       |   REDISSLAVE   |
|   REDIS      |        |       |                |
|              |        |       |                |
|172.22.1.10   |        |       | 172.22.1.20    |
|              |        |       |                |
+--------------+        |       +----------------+
                        |
                        |
                        |       +--------------------+
+---------------+       |       |redissentinelslave  |
|  redissentinel|       |       |                    |
|               |       |       |172.22.1.21         |
|172.22.1.11    |       |       |                    |
|               |       |       +--------------------+
+---------------+       |
                        |
                        |
                        |
+---------------+       |      +----------------+
|redissentinelslavetwo  |      |redissentinelslavethree
|               |       |      |                |
|172.22.1.22    |       |      |172.22.1.23     |
|               |       |      |                |
+---------------+       |      +----------------+
                        |
                        |
+---------------+       |
|express-gateway|       |      +---------------+
|               |       |      |express-gatewaytwo
|               |       |      |               |
|172.22.1.2     |       |      |               |
+---------------+       |      |172.22.1.3     |
                        |      +---------------+
                        |
                        +

splitbrain.js

const Docker = require('dockerode')
const {
    execSync
} = require('child_process');
const IP_PREFIX = '172.22.1.'
var docker = new Docker({
    socketPath: '//./pipe/docker_engine'
});


const action = process.argv[2] ? process.argv[2] : 'add' //delete
console.log('action ' + action)
let group1 = process.argv[3] ? JSON.parse(process.argv[3]) : [10, 11, 12, 13, 2]
let group2 = process.argv[4] ? JSON.parse(process.argv[4]) : [20, 21, 22, 23, 3]

let toInstall = process.argv[5] ? process.argv[5]:false


let block = (id, ip) => {
    let cmd = `docker exec --privileged ${id} route ${action} -host ${ip} reject`
    console.log(cmd)
    execSync(cmd)
}

let install = (id) => {
        let cmd = `docker exec ${id} apt update`
        console.log(cmd)
        let cmd2 = `docker exec ${id} apt install net-tools -y`
        console.log(cmd2)
    try {    
        execSync(cmd)
        execSync(cmd2)
    } catch (e) {}
}

docker.listContainers(function (err, containers) {
    let list = {}

    containers.forEach(element => {
        list[element.NetworkSettings.Networks['express-gateway_egnet'].IPAddress] = element.Id
        if(toInstall) install(element.Id)
    });


    group1.forEach(un => {
        group2.forEach(deux => {
            block(list[IP_PREFIX + un], IP_PREFIX + deux)
            block(list[IP_PREFIX + deux], IP_PREFIX + un)
        });
    });

});