Last active
May 16, 2022 10:10
-
-
Save monossido/9dabfa705181394a1eaa8872e1e43101 to your computer and use it in GitHub Desktop.
Using Redis with Nodejs, MongoDB and mongoose. With normal Query and also Aggregate.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
* Based on @subhrapaladhi tutorial | |
* https://github.com/subhrapaladhi/Node-Redis-Mongo-app https://subhrapaladhi.medium.com/using-redis-with-nodejs-and-mongodb-28e5a39a2696 | |
* I've changed some logic (I'm not returning the data as mongoose Model due to performance concern) | |
* and includedd the support for Aggregation. | |
* Also I'm using the redis 4.* command. | |
*/ | |
const mongoose = require("mongoose"); | |
const redis = require("redis"); | |
const config = require("./../../config.js"); | |
const client = redis.createClient({ url: config.redisUrl }); | |
// We are storing the default exec() function in the exec variable | |
const aggregateExec = mongoose.Aggregate.prototype.exec; | |
const exec = mongoose.Query.prototype.exec; | |
mongoose.Query.prototype.cache = function (hkey) { | |
this.useCache = true; | |
// this is the top level key like motercycles or cars or trucks etc | |
this.hashkey = JSON.stringify(hkey || ""); | |
return this; | |
}; | |
mongoose.Query.prototype.exec = async function () { | |
// Modifing the exec property of mongoose | |
// this = mongoose.Query.prototype.exec | |
// When useCache = false we should directly send the query to MongoDB and return the result to app.js | |
if (!this.useCache) { | |
return exec.apply(this, arguments); | |
} | |
/* Here is how our key looks | |
* key = '{query_param_1: param_1_value, query_param_2: param_2_value,...... , collectoin: collection name}' | |
* we need to stringigy the object before storing in redis cache | |
*/ | |
let key = JSON.stringify( | |
Object.assign({}, this.getQuery(), { | |
collection: this.mongooseCollection.name, | |
}) | |
); | |
/* Querying the cache | |
* if value for key exists then, cacheValue = data | |
* else, cacheValue = null | |
*/ | |
const cacheValue = await client.hGet(this.hashkey, key); | |
// When data is found in redis cache | |
if (cacheValue) { | |
console.log("Data from REDIS!!"); | |
const doc = JSON.parse(cacheValue); // converting back to original datatype from string | |
/* While storing data in redis we may store a single object or an array of objects. | |
* We need to convert normal json into mongoose model instance before returning to app.js, | |
* this.model() is used for this purpose | |
*/ | |
//return Array.isArray(doc) | |
// ? doc.map((d) => new this.model(d)) | |
// : new this.model(doc); | |
// | |
/* Monossido's change. | |
* If we return mongoose model here the performance are bad, almost as querying the db. | |
* I'm returning the normal json obj. We must remember this in the controller. | |
*/ | |
return doc; | |
} | |
// Data not present in redis cache, get the data from Mongodb and save the data to redis cache also | |
const result = await exec.apply(this, arguments); // using the default exec function | |
// just some logic to check if the data for the required query is even present in the database | |
if (result) { | |
console.log("Query Data from DB"); | |
// mongodb retured non-null value (can be empty array) | |
if (Array.isArray(result) && result.length == 0) { | |
// array is empty | |
return null; | |
} else { | |
// data is there (non-empty array or an single object) | |
client.hSet(this.hashkey, key, JSON.stringify(result)); // saving data in redis cache | |
return result; | |
} | |
} else { | |
// database returned null value | |
console.log("Data not present in the DB"); | |
return null; | |
} | |
}; | |
/* | |
* We do almost the same thing also for the aggregate | |
*/ | |
mongoose.Aggregate.prototype.cache = function (hkey) { | |
this.useCache = true; | |
this.hashkey = JSON.stringify(hkey || ""); | |
return this; | |
}; | |
mongoose.Aggregate.prototype.exec = async function () { | |
if (!this.useCache) { | |
return aggregateExec.apply(this, arguments); | |
} | |
let key = JSON.stringify( | |
Object.assign({}, this._pipeline, { | |
collection: this._model.collection.name, | |
}) | |
); | |
const cacheValue = await client.hGet(this.hashkey, key); | |
if (cacheValue) { | |
console.log("Aggregate Data from REDIS!!"); | |
const doc = JSON.parse(cacheValue); | |
return doc; | |
} | |
const result = await aggregateExec.apply(this, arguments); // using the default exec function | |
if (result) { | |
console.log("Aggregate Data from DB"); | |
if (Array.isArray(result) && result.length == 0) { | |
return null; | |
} else { | |
client.hSet(this.hashkey, key, JSON.stringify(result)); // saving data in redis cache | |
return result; | |
} | |
} else { | |
console.log("Aggregate Data not present in the DB"); | |
return null; | |
} | |
}; | |
module.exports = { | |
connect: function () { | |
client.connect(); | |
client.on("connect", () => { | |
console.log("Redis connected"); | |
client.sendCommand(["info", "memory"]).then((r) => { | |
used_memory_human = r.substring( | |
r.indexOf("used_memory_human") + 18, | |
r.indexOf("used_memory_rss") - 2 | |
); | |
console.log("Redis current used_memory_human:", used_memory_human); | |
}); | |
}); | |
}, | |
clearCache: function (hashkey) { | |
console.log("Redis clear Cache about:", hashkey); | |
client.del(JSON.stringify(hashkey)); | |
}, | |
clearAllCache: function () { | |
console.log("Redis clearAllCache"); | |
client.flushAll("ASYNC"); | |
}, | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
FYI @subhrapaladhi