Python script to make diff for same k:v between different redis db's

check.py

#!/usr/local/bin/python3
# -*- coding: utf-8 -*-
from __future__ import print_function

import redis
import json
import time

from itertools import zip_longest
from clint.arguments import Args
from clint.textui import puts, colored, indent
from jsondiff import diff
from argparse import ArgumentParser
from dataknead import Knead

db1 = redis.Redis(host='localhost', port=6379, db=0)
db2 = redis.Redis(host='localhost', port=6379, db=1)

totalEntity = 0

class Range(object):
    def __init__(self, start, end):
        self.start = start
        self.end = end
    def __eq__(self, other):
        return self.start <= other <= self.end

parser = ArgumentParser()

# argument to check specific key
parser.add_argument("-k", "--key", dest="key", default='*',
                    help="Specifies key for check integration")

# argument to define quantity of keus returned per batch
parser.add_argument("-b", "--batch", dest="batch", default='1000',
                    help="Limit records per batch")

# define sleep for every each entity
# to better read the outputs, pass a high value between 0.0 and 5.0
parser.add_argument("-sl", "--sleep", dest="sleep", default=0.0, type=float,
                    choices=[Range(0.0, 5.0)], help="Sleep every each record, choise between 0.0 and 5.0")

args = parser.parse_args()

key = args.key
limitPerBatch = args.batch
sleep = args.sleep

# iterate a list in batches of size n
def batcher(iterable, n):
    args = [iter(iterable)] * n
    return zip_longest(*args)

for keysBatch in batcher(db1.scan_iter(key), int(limitPerBatch)):
    with indent(2, quote='>'):
        puts(colored.blue('Processing Batcher: ') + len(keysBatch))

    for key in keysBatch:

        # same times key is none type
        if key is None:
            continue

        totalEntity += 1

        with indent(3, quote='>>'):
            puts(colored.blue('Key: ') + key.__str__())

        # get entity in development env
        entityDb1 = json.loads(db1.get(key))

        with indent(4, quote='>>>'):
            # try to catch same entity for db2
            try:
                entityDb2 = json.loads(db2.get(key))

                # make diff in all json payload
                jsonDiff = json.loads(diff(entityDb1, entityDb2, syntax='symmetric', dump=True))

                # print json diff
                puts(colored.yellow('Diff Between k:v: ') + json.dumps(jsonDiff, indent=4, sort_keys=True))

            except TypeError as e:
                puts(colored.red('Same key not found in db2 cache. Going to next...'))
                continue

        # just sleep for system
        # and human read
        time.sleep(sleep)

puts(colored.blue('Total Entity: ') + totalEntity)

A simple script to check integrity of the same k:v between different db. It can assist in large-scale integrity checking when changing a property value, business rule or remove property.

Spected output for each k:v :

> Processing Batcher: 1000
>> Key: b'ec:product:DI765TB0T7RFCLAEC'
>>> Diff Between k:v : {
>>>     "simples": {
>>>         "0": {
>>>             "$insert": {
>>>                 "memberships": null
>>>             }
>>>         }
>>>     },
>>>     "updated_at": [
>>>         "2019-03-04 18:54:20",
>>>         "2019-03-04 18:59:13"
>>>     ]
>>> }

Script use jsondiff to output diff between entity payload.

e.g:

check.py -sl 1 -k 'key:*'