Redis Cluster migration Script

redis_cluster_migration.py

#Working Solution with the runtime of 2:30 mins
from rediscluster import RedisCluster
from multiprocessing import Pool

# Connect to source Redis cluster
src_redis = RedisCluster(startup_nodes=[{"host": "xxxx.xxxxxx.clustercfg.xxxx.cache.amazonaws.com", "port": "6379"}], decode_responses=True,skip_full_coverage_check=True)

# Connect to destination Redis cluster
dst_redis = RedisCluster(startup_nodes=[{"host": "xxxx.xxxxxx.clustercfg.xxxx.cache.amazonaws.com", "port": "6379"}], decode_responses=True,skip_full_coverage_check=True)

# Define a function to migrate keys from one cluster to another
def migrate_keys(src_keys):
    try:
        str_value = src_redis.get(src_keys)
        dst_redis.set(src_keys, str_value)
    except Exception as e:
        key_type = src_redis.type(src_keys)
        if(key_type == 'list'):
            list_value = src_redis.lrange(src_keys, 0, -1)
            for data in list_value:
                dst_redis.rpush(src_keys, data)
        elif(key_type == 'hash'):
            hash_value = src_redis.hgetall(src_keys)
            dst_redis.hset(src_keys, mapping=hash_value)
        else:
            print("Invalid key type: ", key_type,"\nCopying skipped")

# Get all keys from source Redis cluster
source_keys = src_redis.scan_iter("*")

# Create a pool of worker processes
pool = Pool(processes=10)

print("Migration started!")

# Use map function to migrate keys in parallel
pool.map(migrate_keys, source_keys)

# Close the pool
pool.close()
pool.join()

print("Migration completed!")