Oddworks Datastore Scan Tests

test-redis-scan.js

'use strict';

const Promise = require('bluebird');
const _ = require('lodash');
const redis = require('redis');

const REDIS_SCAN_PAGES = 5;
const PAGE_SIZE = 20;

Promise.promisifyAll(redis.RedisClient.prototype);
Promise.promisifyAll(redis.Multi.prototype);

exports.main = function (args) {
	const url = args.redisUrl;
	const client = redis.createClient({url});

	const PATTERN = 'crtv:video:*';

	function getPage(pages, results, cursor, pageSize) {
		return client.scanAsync(cursor, 'MATCH', PATTERN, 'COUNT', pageSize).then(res => {
			pages += 1;
			results = results.concat(res[1]);
			cursor = parseInt(res[0], 10);

			if (cursor) {
				return getPage(pages, results, cursor, pageSize);
			}

			return {pages, results};
		});
	}

	function hscan(keys) {
		const commands = keys.map(key => {
			return ['hscan', key, 0, 'COUNT', 256];
		});

		return client.multi(commands).execAsync().then(results => {
			return _.flatten(results.map(res => {
				return res[1].filter((item, i) => {
					return i % 2 === 1;
				});
			}));
		});
	}

	const start = Date.now();
	let pageSize;
	let keys;
	let redisPages;
	let results;

	console.log(' --- start ---');

	return Promise.resolve(null)
		.then(() => {
			return client.dbsizeAsync().then(res => {
				pageSize = Math.ceil(res / REDIS_SCAN_PAGES);
				return null;
			});
		})
		.then(() => {
			return getPage(0, [], 0, pageSize).then(res => {
				keys = res.results;
				redisPages = res.pages;
				return null;
			});
		})
		.then(() => {
			const time = Date.now() - start;
			console.log('Pattern:', PATTERN);
			console.log('Total keys:', keys.length);
			console.log('Total Redis pages required:', redisPages);
			console.log('Redis Page size', pageSize);
			console.log('Time required to get keys:', time);
			return null;
		})
		.then(() => {
			return hscan(keys.slice(0, PAGE_SIZE)).then(items => {
				results = JSON.parse(`[${items.join(',')}]`);
			});
		})
		.then(() => {
			const time = Date.now() - start;
			console.log('Page size', PAGE_SIZE);
			console.log('Results:', results.length);
			console.log('Time required to get results:', time);
		})
		.then(() => {
			client.end(false);
			return Promise.delay(300);
		});
};

if (require.main === module) {
	const redisUrl = process.env.REDIS_URL;

	if (!redisUrl) {
		throw new Error('REDIS_URL env var must be set');
	}

	exports.main({redisUrl}).then(() => {
		console.log(' --- done ---');
	}).catch(err => {
		console.log(' --- done; with error ---');
		console.error(err.stack);
	});
}