Catastrophe recover for web3.storage by repopulating a new ipfs-cluster with backup data stored

web3-storage-catastrophe-recover-repopulate-ipfs-cluster.js

import dotenv from 'dotenv'
import {
  S3Client,
  paginateListObjectsV2,
  GetObjectCommand
} from '@aws-sdk/client-s3'
import { Cluster } from '@nftstorage/ipfs-cluster'

dotenv.config({ path: '.env.local' })

async function main() {
  // Setup S3 client
  const region = process.env.S3_BUCKET_REGION
  const bucketName = process.env.BUCKET_NAME
  const accessKeyId = process.env.S3_ACCESS_KEY_ID
  const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY_ID

  if (!region || !accessKeyId || !secretAccessKey || !bucketName) {
    throw new Error('S3_BUCKET_REGION, BUCKET_NAME, S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY_ID environment variables must be specified')
  }

  const s3Client = new S3Client({
    region,
    credentials: {
      accessKeyId,
      secretAccessKey
    }
  })

  // Setup IPFS Cluster
  const clusterApiUrl = process.env.CLUSTER_API_URL
  const clusterAuthToken = process.env.CLUSTER_BASIC_AUTH_TOKEN

  if (!clusterApiUrl) {
    throw new Error('CLUSTER_API_URL environment variable must be specified')
  }

  const headers = clusterAuthToken ? { Authorization: `Basic ${clusterAuthToken}` } : {}
  const cluster = new Cluster(clusterApiUrl, { headers })

  // Recover data
  for await (const data of paginateListObjectsV2({ client: s3Client, pageSize: 10 })) {
    // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/listobjectsv2commandoutput.html#contents
    // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/modules/_object.html
    await Promise.all(data.Contents.map(
      ({ key }) => dataRecover(key, bucketName, s3Client, cluster))
    )
  }
}

/**
 * @param {string} key
 * @param {string} bucketName
 * @param {S3Client} s3Client
 * @param {Cluster} cluster
 */
async function dataRecover (key, bucketName, s3Client, cluster) {
  const command = new GetObjectCommand({
    Bucket: bucketName,
    Key: key
  })
  try {
    const s3Item = await s3Client.send(command)
    await cluster.add(s3Item.Body) // TODO: see options
  } catch (err) {
    console.log('error: ', err)
  }
}

main()