Created
October 3, 2024 19:03
-
-
Save 45deg/6b4b01477cdac6dd8f600285e20c3f21 to your computer and use it in GitHub Desktop.
pglite with pgvector example code with file dump/load
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import { PGlite } from "@electric-sql/pglite"; | |
import { vector } from '@electric-sql/pglite/vector'; | |
import fs from 'node:fs/promises'; | |
import { createReadStream } from "node:fs"; | |
import zlib from 'node:zlib'; | |
import { pipeline } from 'node:stream/promises'; | |
const pg = new PGlite({ | |
extensions: { vector }, | |
}); | |
await pg.sql`CREATE EXTENSION IF NOT EXISTS vector`; | |
// init the database | |
await pg.sql`CREATE TABLE IF NOT EXISTS items (id bigserial PRIMARY KEY, embedding vector(3))`; | |
await pg.sql`CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops)`; | |
//await pg.sql`INSERT INTO items (embedding) VALUES ('[1, 2, 3]')`; | |
// insert random | |
for (let i = 0; i < 1000; i++) { | |
await pg.sql`INSERT INTO items (embedding) VALUES (${`[${Math.random()}, ${Math.random()}, ${Math.random()}]`})`; | |
} | |
// query the database | |
await fs.writeFile('foo.gz', (await pg.dumpDataDir('gzip')).stream()); | |
const q = `[${Math.random()}, ${Math.random()}, ${Math.random()}]` | |
console.log(await pg.sql`SELECT * FROM items ORDER BY embedding <-> ${q} LIMIT 10`); | |
// another instance | |
await pg.close(); | |
const chunks = []; | |
await pipeline( | |
createReadStream('foo.gz'), | |
zlib.createGunzip(), | |
async function* (source) { | |
for await (const chunk of source) { | |
chunks.push(chunk); | |
} | |
} | |
) | |
const pg2 = new PGlite({ | |
extensions: { vector }, | |
loadDataDir: new Blob(chunks), | |
}); | |
console.log(await pg2.sql`SELECT * FROM items ORDER BY embedding <-> ${q} LIMIT 10`); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment