Last active
May 14, 2018 04:10
-
-
Save Lwdthe1/5b6931fc86e4c84f63b7e7e0bcb2d5f9 to your computer and use it in GitHub Desktop.
A function to page objects from MongoDb using Mongoose. It's pretty general, so the basic structure can be used in implementing paging on any database. I use this implementation to page some resources at https://www.smedian.com
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Schema.statics._countByQuery = function(query) { | |
if(!query) { | |
throw new Error("query object is required") | |
} | |
return this.count(query) | |
.fail((err) => { | |
console.warn(err.message) | |
return Q.resolve(null) | |
}) | |
} | |
/** | |
* Finds instances of the provided schema with paging values | |
* @param {{ | |
findQuery: {Object} query to use in finding the bare objects for the current batch, | |
skipQuery: {Object} query to use when counting the number of objects to skip when fetching the current batch, | |
getRemainderQuery: {(Array<DbObject>) => Promise<Array<DbObject>>} function to get the remaining db objects after finding the current batch, | |
sort: {(Array<DbObject>) => Promise<Array<DbObject>>} function to sort the objects we found the current batch, | |
bareSelectFields: {?Array<string>} array of fields to include on the bare objects that we find for the current batch, | |
limit: {?Number}, the maximum number of objects to find for the current batch, | |
sortQuery: {?Object} the query to use to sort the objects we find for the current batch, | |
getBatch: {({ids: Array<string>, dbObjectsBare: Array<Object>})} gets the objects for the current batch, | |
getNextBatchPagingFromValue: {({lastObject: Object }) => {number|string|null}} | |
}} config | |
* | |
* @returns {{ | |
dbObjects: {Array<DbObject>} The full objects found for the current batch, | |
remainder: {Number} The remaining objects that did not fit in the current batch, | |
from: {number|string|null} The starting marker for counting the objects to skip when fetching the next batch. | |
}} | |
*/ | |
Schema.statics._findWithPaging = function({ | |
findQuery, skipQuery, getRemainderQuery, sortQuery, | |
sort, bareSelectFields, limit, getBatch, getNextBatchPagingFromValue | |
}) { | |
if (!findQuery) throw new Error('findQuery object is required') | |
if (!sortQuery) throw new Error('sortQuery object is required') | |
if (!getBatch) throw new Error('getBatch function is required') | |
if (!getRemainderQuery) throw new Error('getRemainderQuery function is required') | |
if (!getNextBatchPagingFromValue) throw new Error('getNextBatchPagingFromValue function is required') | |
sm.try(() => { | |
limit = parseInt(limit) | |
}) | |
if (!limit) limit = 10 | |
// count the number of objects to skip when fetching the current batch | |
// presumably, those objects have already been fetched | |
let skipCountPromise = Q.resolve(0) | |
if (skipQuery) { | |
skipCountPromise = this._countByQuery(skipQuery) | |
} | |
// ensure the select fields we want | |
if (!bareSelectFields) bareSelectFields = [] | |
bareSelectFields.push('_id') | |
bareSelectFields.push('createdAt') | |
// get the number of objects to skip | |
return skipCountPromise | |
.then((count) => { | |
// find the bare objects | |
let finder = this.find(findQuery) | |
.lean(true) | |
.select(bareSelectFields.reduce((map, field) => { | |
map[field] = 1 | |
return map | |
}, {})) | |
.skip(count) | |
.limit(limit) | |
// sort the objects in the db driver | |
if (sortQuery) { | |
finder.sort(sortQuery) | |
} | |
return finder | |
.exec() | |
.then((dbObjectsBare) => { | |
if (sortQuery || !sort) return dbObjectsBare | |
// sort the objects with the provided function | |
return sort(dbObjectsBare) | |
}) | |
}) | |
.then((dbObjectsBare) => { | |
if(sm.utils.arrays.isEmptyArray(dbObjectsBare)) { | |
return Q.resolve({ dbObjects: [], remainder: 0 }) | |
} | |
let ids = sm.dbManager.pluckIds(dbObjectsBare) | |
let total = dbObjectsBare.length | |
let nextBatchFromValue | |
let remainderPromise | |
if (total) { | |
let lastObject = dbObjectsBare[total - 1] | |
nextBatchFromValue = getNextBatchPagingFromValue({lastObject}) | |
let remainderQuery = getRemainderQuery({ | |
nextBatchFromValue: nextBatchFromValue, | |
dbObjectsBare: dbObjectsBare, | |
lastObject: lastObject, | |
total: total, | |
}) | |
// get the count of remaining objects | |
if (remainderQuery) { | |
remainderPromise = this._countByQuery(remainderQuery) | |
} | |
} | |
if (!remainderPromise) remainderPromise = Q.resolve(0) | |
// get the final objects and | |
return Q.all([getBatch({ids: ids, dbObjectsBare: dbObjectsBare}), remainderPromise]) | |
.spread((dbObjects, remainder) => { | |
return { dbObjects: dbObjects, remainder: remainder, from: nextBatchFromValue } | |
}) | |
}) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment