Last active
November 20, 2018 00:09
-
-
Save ChrisCinelli/ddae2b9db5f96db6e8a53059fe909e0e to your computer and use it in GitHub Desktop.
cache for npm request and elastic search
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var _request = require('request'); | |
const { stringify } = require('flatted/cjs'); | |
const LRU = require('lru-cache'); | |
const crypto = require('crypto'); | |
// Interface for cached requests | |
class Cache { | |
// opt to pass to LRU, h hash function for the key | |
constructor({ opts = {}, h = (k) => k, Type = LRU }) { | |
this.cache = new Type(opts); | |
this.h = h; | |
} | |
// Return promisified object | |
getOrElse (key, valueF, opts = {}) { | |
const _key = this.h(key); | |
const cachedValue = this.cache.get(_key); | |
if (cachedValue) return cachedValue; | |
const pV = valueF(); | |
this.cache.set(_key, pV, opts); | |
return pV; | |
} | |
} | |
function md5(obj) { | |
return crypto.createHash('MD5').update(stringify(obj)).digest('hex'); | |
} | |
class CachedRequest { | |
constructor({ opts = {}}) { | |
this.cache = new Cache({ opts, h: md5}); | |
this.opts = opts; | |
this.getCache = this.getCache.bind(this); | |
this.request = this.request.bind(this); | |
} | |
getCache() { | |
return this.cache; | |
} | |
request(url, cb) { | |
this.cache.getOrElse(url, () => { | |
return new Promise((resolve, reject) => { | |
//console.log('--------> Requesting', url), | |
_request(url, function(err, result, body) {resolve([err, result, body]);} ); | |
}) | |
}).then(r => cb.apply(this, r)); | |
} | |
} | |
module.exports = { | |
md5, | |
Cache, | |
CachedRequest | |
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const elasticsearch = require('elasticsearch'); | |
const configHelper = require('./configHelper'); | |
const { Cache, md5 } = require('./cachedRequest'); | |
let esPromise; | |
let intervalId; | |
// When we need a ES connection we use getES() | |
// Both conenctions and searches are cached | |
const defaultOptions = { | |
esHost: 'esHost', | |
max: 10000, // max: max entries in the cache (default: 10000) | |
maxAge: undefined, // to set max cache age in ms (default: none) | |
cacheResetTime: 10 * 60 * 1000 // to set max cache age in ms (default: 10 minutes) | |
}; | |
const pub = module.exports = { | |
opts: { | |
esHost: 'esHost', | |
max: 10000, // max: max entries in the cache (default: 10000) | |
maxAge: undefined, // to set max cache age in ms (default: none) | |
cacheResetTime: 10 * 60 * 1000 // to set max cache age in ms (default: 10 minutes) | |
}, | |
setOpt: function(opts) { | |
pub.opts = {...defaultOptions, ...opts}; | |
}, | |
getES: function() { | |
if (!esPromise) { | |
esPromise = configHelper.getConfigValue(pub.opts.esHost).then(hostsFromConfig => { | |
let hosts = hostsFromConfig; | |
//Syntax for one host: username + ':' + password + '@' + host | |
if (typeof(hostsFromConfig) === 'string') hosts = [hosts]; | |
var es = new elasticsearch.Client({ | |
hosts | |
}) | |
es.oldSearch = es.search; | |
const es_results_cache = new Cache({ h: md5, opts: {...pub.opts } }); | |
es.search = function(what) { | |
return es_results_cache.getOrElse(what, () => { | |
// console.log('ES request:', JSON.stringify(what, null, 2)) | |
return es.oldSearch(what) | |
}); | |
} | |
return es; | |
}); | |
} | |
return esPromise; | |
}, | |
resetES: function () { | |
esPromise = undefined; | |
}, | |
resetESInterval: function (time) { | |
if (intervalId) clearInterval(intervalId); | |
intervalId = setInterval(() => this.resetES(), pub.opts.cacheResetTime); // Default 10 minutes; | |
} | |
} | |
pub.resetESInterval(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment