Created
March 2, 2023 01:10
-
-
Save joewagner/f240b91246081dbd1202b13a0714f802 to your computer and use it in GitHub Desktop.
An example of helpers for Tableland bulk inserts that will need to be split into multiple transactions.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Dependencies: @tableland/sdk, @tableland/local | |
// Run `npx local-tableland` and then run this script in another window | |
import { Database, helpers } from "@tableland/sdk"; | |
import { Wallet, getDefaultProvider, utils } from "ethers"; | |
// NOTE: using 34000 here to give a 1000 byte padding for the `insert into table values` and | |
// in case the sql parser increases the size, which is unlikely but possible. | |
// If you wanted to limit for gas you'd have to kinda guess a value here. | |
const BYTES_LIMIT = 34000 | |
async function main() { | |
const db = getDb(); | |
const { meta: create } = await db | |
.prepare( | |
`CREATE TABLE drivers (permit_id integer primary key, race_team text, is_holo int, is_revealed int, driver_id int, permit_pfp_address text, permit_pfp_id int, is_discord_contributor int);` | |
) | |
.run(); | |
const { name: tableName } = create.txn; | |
console.log(`Table name: ${tableName}`); | |
const formatValues = generateValues(); | |
const results = await helpBatchBulkInsert({ | |
tableName, | |
values: formatValues, | |
schema: "permit_id,race_team,is_holo,is_revealed,driver_id", | |
db, | |
bytesPerChunk: BYTES_LIMIT | |
}) | |
// I randomized the choice of team, so these will be close to equal but probably not exactly. | |
console.log("team 1", (await db.prepare(`select count(*) from ${tableName} where race_team = 'team 1';`).all()).results[0]["count(*)"]); | |
console.log("team 2", (await db.prepare(`select count(*) from ${tableName} where race_team = 'team 2';`).all()).results[0]["count(*)"]); | |
console.log("team 3", (await db.prepare(`select count(*) from ${tableName} where race_team = 'team 3';`).all()).results[0]["count(*)"]); | |
console.log("team 4", (await db.prepare(`select count(*) from ${tableName} where race_team = 'team 4';`).all()).results[0]["count(*)"]); | |
} | |
const byteSize = str => new Blob([str]).size; | |
function helpGetInsertChunks(valuesStrings, bytesPerChunk) { | |
const chunks = [] | |
while (valuesStrings.length > 0) { | |
let chunk = ""; | |
while (valuesStrings.length > 0 && byteSize(chunk) < bytesPerChunk) { | |
// My intuition was fifo, but maybe filo makes more sense? | |
chunk += `${valuesStrings.shift()},`; | |
} | |
// if a single values | |
if (!chunk) throw new Error("could not chunk the values"); | |
// slice the last comma off | |
chunks.push(chunk.slice(0, -1)); | |
} | |
return chunks; | |
} | |
async function helpBatchBulkInsert(params) { | |
const chunks = helpGetInsertChunks(params.values, params.bytesPerChunk); | |
const responses = [] | |
for (const chunk of chunks) { | |
// Since the db is using autoWait this won't return until the transaction is finished. | |
const res = await params.db.prepare( | |
`INSERT INTO ${params.tableName}(${params.schema})VALUES${chunk}` | |
).all(); | |
responses.push(res) | |
if (!res.success) { | |
console.warn("insert failed"); | |
// Now what? | |
// - Maybe you want to stop on the first failure? | |
// - Maybe you want log the failures then retry them separately? | |
} | |
// Note that `res.blockNumber` will be incrementing by 3 each time because we | |
// are waiting for the 3 blocks before the transaction is considered valid. | |
console.log(res); | |
} | |
return responses; | |
} | |
function getDb() { | |
// Hard code Hardhat private key | |
const privateKey = | |
"0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a"; | |
// Define the signer and connect to the provider. | |
const wallet = new Wallet(privateKey); | |
// A local Hardhat node from running `npx local-tableland`, but replace | |
// with any provider URL (e.g., Alchemy, Infura, Etherscan, etc.). | |
const provider = getDefaultProvider("http://127.0.0.1:8545"); | |
const signer = wallet.connect(provider); | |
return new Database({ | |
// If we wait until transactions are finished before returning | |
// it makes reasoning about control flow easier. | |
autoWait: true, | |
signer | |
}); | |
} | |
const teams = ["team 1", "team 2", "team 3", "team 4"]; | |
// This will probably come from a file or something a bit more intentional. | |
function generateValues() { | |
return Array(8000).fill().map( | |
function (_, i) { | |
const team = teams[Math.floor(Math.random() * teams.length)]; | |
return `(${i},'${team}',0,1,${i})`; | |
} | |
); | |
} | |
await main(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment