Hypercore download speed

create.js

// npm i hypercore-id-encoding corestore hyperswarm b4a tiny-byte-size repl-swarm graceful-goodbye
// node create.js
// Change the constants to change the hypercore size

const idEnc = require('hypercore-id-encoding')
const Corestore = require('corestore')
const Hyperswarm = require('hyperswarm')
const b4a = require('b4a')
const formatBytes = require('tiny-byte-size')
const replSwarm = require('repl-swarm')
const goodbye = require('graceful-goodbye')

const blockSizeBytes = 65536
const coreLength = 15250 // around 1Gb
const corestoreLoc = 'create-corestore'
const exposeRepl = true

async function main () {
  const store = new Corestore(corestoreLoc)
  const swarm = new Hyperswarm()
  swarm.on('connection', (conn) => {
    store.replicate(conn)
  })

  const core = store.get({ name: `core-${coreLength}-${blockSizeBytes}` })

  if (exposeRepl === true) {
    const replKey = replSwarm({ core, store, swarm })
    console.warn(`Exposed repl swarm at key ${replKey} (core, store, swarm)`)
  }

  goodbye(async () => {
    try {
      console.info('Shutting down')
      await swarm.destroy()
      console.info('swarm shut down')
      await store.close()
    } catch (e) {
      console.error(`Error while shutting down ${e.stack}`)
    }
  })

  await core.ready()

  if (core.length === coreLength) {
    console.info('Found existing core')
  }

  for (let i = core.length; i < coreLength; i++) {
    if (i % 10000 === 0) console.info(`Added block ${i}`)
    await core.append(b4a.allocUnsafe(blockSizeBytes)) // We don't really care about the block content
  }

  swarm.join(core.discoveryKey, { client: false, server: true })

  const info = await core.info()
  if (info.length !== coreLength) {
    throw new Error(`Logical bug: created core with other length than the expected ${coreLength} (${info.length})`)
  }
  if (info.byteLength !== coreLength * blockSizeBytes) {
    throw new Error(`Logical bug: created core with other byteLength than the expected ${coreLength * blockSizeBytes} (${info.byteLength})`)
  }

  console.info(`Started serving core of ${formatBytes(info.byteLength)} with ${info.length} blocks of ${formatBytes(blockSizeBytes)}`)
  console.info(`Public key ${idEnc.normalize(core.key)} (Discovery key: ${idEnc.normalize(core.discoveryKey)})`)
}

main()

seed.js

// Downloads + seeds a core (measures download time)
// npm i hypercore-id-encoding corestore hyperswarm repl-swarm graceful-goodbye
// node seed.js <pubkey>
// For multiple runs on same machine:
// node seed.js <pubkey> <runId>
//  For example:
//    node seed.js p44cpkjahjm7s9i1651nzxc6ufh5dhwaxj53cfq4gun39rjnwphy 2
//    node seed.js p44cpkjahjm7s9i1651nzxc6ufh5dhwaxj53cfq4gun39rjnwphy 3

const idEnc = require('hypercore-id-encoding')
const Corestore = require('corestore')
const Hyperswarm = require('hyperswarm')
const replSwarm = require('repl-swarm')
const goodbye = require('graceful-goodbye')

const exposeRepl = false

async function main () {
  const key = idEnc.normalize(process.argv[2])
  const seedNr = parseInt(process.argv[3] || 1)
  const corestoreLoc = `seed-${seedNr}-corestore`

  console.info(`Using corestore ${corestoreLoc}`)

  const store = new Corestore(corestoreLoc)
  const swarm = new Hyperswarm()

  let nrConnections = 0
  swarm.on('connection', (conn) => {
    nrConnections++
    console.info(`Connected to peer (total ${nrConnections})`)
    conn.on('close', () => {
      nrConnections--
      console.info(`Disconnected from peer (total ${nrConnections})`)
    })
    store.replicate(conn)
  })

  const core = store.get({ key })
  core.on('append', () => {
    console.info(`core updated (length: ${core.length})`)
  })

  console.time('hypercore download')

  let blocksDownloaded = 0
  core.on('download', async () => {
    blocksDownloaded++
    if (blocksDownloaded % 100 === 0) {
      console.timeLog('hypercore download', `Downloaded ${blocksDownloaded} blocks`)
    }
    if (core.contiguousLength === core.length) {
      console.info('Core fully downloaded')
      console.timeEnd('hypercore download')
    }
  })

  if (exposeRepl === true) {
    const replKey = replSwarm({ core, store, swarm })
    console.warn(`Exposed repl swarm at key ${replKey} (core, store, swarm)`)
  }

  goodbye(async () => {
    try {
      console.info('Shutting down')
      await swarm.destroy()
      console.info('swarm shut down')
      await store.close()
    } catch (e) {
      console.error(`Error while shutting down ${e.stack}`)
    }
  })

  await core.ready()

  swarm.join(core.discoveryKey, { client: true, server: true })

  core.download({ start: 0, end: -1 })

  console.info(`Started downloading core ${key} (current length: ${core.length})`)

  if (core.length > 1 && core.contiguousLength === core.length) {
    console.info('Core was already fully downloaded')
  }
}

main()