Skip to content

Instantly share code, notes, and snippets.

@arshamalh
Created December 13, 2021 07:03
Show Gist options
  • Save arshamalh/2fb932fcfdffc184a30036623a4878be to your computer and use it in GitHub Desktop.
Save arshamalh/2fb932fcfdffc184a30036623a4878be to your computer and use it in GitHub Desktop.
Cursor pagination in PostgreSQL and Nodejs
// Written by Arsham Arya
// I'm not sure it's the best way,
// So any contribution makes me and other readers happy :)
import pgPromise from "pg-promise";
import Cursor from "pg-cursor";
require("dotenv").config();
const pgp = pgPromise();
const db = pgp({
user: process.env.PG_USER,
password: process.env.PG_PASS,
host: process.env.PG_HOST,
port: process.env.PG_PORT,
database: process.env.PG_DB,
});
async function connectToPG() {
return new Promise((resolve, reject) => {
db.connect().then((connection) => {
resolve(connection);
})
})
}
async function getData(connection, batchSize = 1000, startFrom = 0, callback) {
let cursor = connection.client.query(new Cursor(
`SELECT * FROM table_name`
))
let count = 1;
let rows = await cursor.read(batchSize);
if (rows.length && count >= startFrom) {
callback(rows);
}
while (rows.length) {
rows = await cursor.read(batchSize);
count++;
if (rows.length && count >= startFrom) {
callback(rows);
}
}
}
connectToPG().then((connection) => {
console.log("Connected to PostgreSQL")
getData(connection, 1000, 0, (data) => {
// Do anything you want with data!
})
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment