Skip to content

Instantly share code, notes, and snippets.

@Lwdthe1
Last active May 14, 2018 04:10
Show Gist options
  • Save Lwdthe1/5b6931fc86e4c84f63b7e7e0bcb2d5f9 to your computer and use it in GitHub Desktop.
Save Lwdthe1/5b6931fc86e4c84f63b7e7e0bcb2d5f9 to your computer and use it in GitHub Desktop.
A function to page objects from MongoDb using Mongoose. It's pretty general, so the basic structure can be used in implementing paging on any database. I use this implementation to page some resources at https://www.smedian.com
Schema.statics._countByQuery = function(query) {
if(!query) {
throw new Error("query object is required")
}
return this.count(query)
.fail((err) => {
console.warn(err.message)
return Q.resolve(null)
})
}
/**
* Finds instances of the provided schema with paging values
* @param {{
findQuery: {Object} query to use in finding the bare objects for the current batch,
skipQuery: {Object} query to use when counting the number of objects to skip when fetching the current batch,
getRemainderQuery: {(Array<DbObject>) => Promise<Array<DbObject>>} function to get the remaining db objects after finding the current batch,
sort: {(Array<DbObject>) => Promise<Array<DbObject>>} function to sort the objects we found the current batch,
bareSelectFields: {?Array<string>} array of fields to include on the bare objects that we find for the current batch,
limit: {?Number}, the maximum number of objects to find for the current batch,
sortQuery: {?Object} the query to use to sort the objects we find for the current batch,
getBatch: {({ids: Array<string>, dbObjectsBare: Array<Object>})} gets the objects for the current batch,
getNextBatchPagingFromValue: {({lastObject: Object }) => {number|string|null}}
}} config
*
* @returns {{
dbObjects: {Array<DbObject>} The full objects found for the current batch,
remainder: {Number} The remaining objects that did not fit in the current batch,
from: {number|string|null} The starting marker for counting the objects to skip when fetching the next batch.
}}
*/
Schema.statics._findWithPaging = function({
findQuery, skipQuery, getRemainderQuery, sortQuery,
sort, bareSelectFields, limit, getBatch, getNextBatchPagingFromValue
}) {
if (!findQuery) throw new Error('findQuery object is required')
if (!sortQuery) throw new Error('sortQuery object is required')
if (!getBatch) throw new Error('getBatch function is required')
if (!getRemainderQuery) throw new Error('getRemainderQuery function is required')
if (!getNextBatchPagingFromValue) throw new Error('getNextBatchPagingFromValue function is required')
sm.try(() => {
limit = parseInt(limit)
})
if (!limit) limit = 10
// count the number of objects to skip when fetching the current batch
// presumably, those objects have already been fetched
let skipCountPromise = Q.resolve(0)
if (skipQuery) {
skipCountPromise = this._countByQuery(skipQuery)
}
// ensure the select fields we want
if (!bareSelectFields) bareSelectFields = []
bareSelectFields.push('_id')
bareSelectFields.push('createdAt')
// get the number of objects to skip
return skipCountPromise
.then((count) => {
// find the bare objects
let finder = this.find(findQuery)
.lean(true)
.select(bareSelectFields.reduce((map, field) => {
map[field] = 1
return map
}, {}))
.skip(count)
.limit(limit)
// sort the objects in the db driver
if (sortQuery) {
finder.sort(sortQuery)
}
return finder
.exec()
.then((dbObjectsBare) => {
if (sortQuery || !sort) return dbObjectsBare
// sort the objects with the provided function
return sort(dbObjectsBare)
})
})
.then((dbObjectsBare) => {
if(sm.utils.arrays.isEmptyArray(dbObjectsBare)) {
return Q.resolve({ dbObjects: [], remainder: 0 })
}
let ids = sm.dbManager.pluckIds(dbObjectsBare)
let total = dbObjectsBare.length
let nextBatchFromValue
let remainderPromise
if (total) {
let lastObject = dbObjectsBare[total - 1]
nextBatchFromValue = getNextBatchPagingFromValue({lastObject})
let remainderQuery = getRemainderQuery({
nextBatchFromValue: nextBatchFromValue,
dbObjectsBare: dbObjectsBare,
lastObject: lastObject,
total: total,
})
// get the count of remaining objects
if (remainderQuery) {
remainderPromise = this._countByQuery(remainderQuery)
}
}
if (!remainderPromise) remainderPromise = Q.resolve(0)
// get the final objects and
return Q.all([getBatch({ids: ids, dbObjectsBare: dbObjectsBare}), remainderPromise])
.spread((dbObjects, remainder) => {
return { dbObjects: dbObjects, remainder: remainder, from: nextBatchFromValue }
})
})
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment