Sample dotStorage CIDs

sample.mjs

/**
 * Create IPFS unixfs directories of random CIDs from the data source. CIDs are
 * grouped by DAG size:
 * 0-1MB
 * 1MB-5MB
 * 5MB-25MB
 * 25MB-100MB
 * 100MB-500MB
 * 500MB-1GB
 * 1GB-10GB
 * 10GB-32GB
 */
import pg from 'pg'
import dotenv from 'dotenv'
import { create } from 'ipfs-http-client'
import bytes from 'bytes'

dotenv.config()

const FETCH_CONTENT_SQL = 'SELECT cid, dag_size FROM content TABLESAMPLE SYSTEM(1) WHERE dag_size IS NOT NULL AND dag_size > $1 AND dag_size < $2 ORDER BY RANDOM() LIMIT $3'

const MiB = 1024 * 1024
const GiB = 1024 * MiB

const sizes = [
  [0, MiB],
  [MiB, 5 * MiB],
  [5 * MiB, 25 * MiB],
  [25 * MiB, 100 * MiB],
  [100 * MiB, 500 * MiB],
  [500 * MiB, GiB],
  [GiB, 10 * GiB]
  // [10 * GiB, 32 * GiB]
]

const LIMIT = 250

async function main () {
  const ipfs = create()
  const client = new pg.Client({ connectionString: process.env.DATABASE_CONNECTION })
  await client.connect()
  console.log('🔌 postgres connected!')

  try {
    const limit = process.env.LIMIT ? parseInt(process.env.LIMIT) : LIMIT
    const rootPath = `/samples-${Date.now()}`
    for (const [min, max] of sizes) {
      const res = await client.query(FETCH_CONTENT_SQL, [min, max, limit * 2])
      if (!res.rows.length) throw new Error('no rows')
      const basePath = `${rootPath}/${bytes(min)}-${bytes(max)}`
      let i = 0
      for (const { cid } of res.rows) {
        console.log(`cp /ipfs/${cid} -> ${basePath}/${cid}`)
        try {
          await ipfs.files.cp(`/ipfs/${cid}`, `${basePath}/${cid}`, { parents: true, timeout: 30000 })
        } catch (err) {
          console.error(err)
          continue
        }
        i++
        if (i >= limit) break
      }
      console.log(`write ${basePath}/manifest.json`)
      await ipfs.files.write(`${basePath}/manifest.json`, JSON.stringify(res.rows.slice(0, i), null, 2), { parents: true, create: true })
      const stat = await ipfs.files.stat(basePath)
      console.log(`stat ${basePath} -> /ipfs/${stat.cid}`)
    }
    const stat = await ipfs.files.stat(rootPath)
    console.log(`stat ${rootPath} -> /ipfs/${stat.cid}`)
  } finally {
    await client.end()
  }
}

main().then(() => console.log('✅ done')).catch(console.error)