Migrate Fauna documents

migrate-fauna-data.ts

/**
 * This script migrates data from a Fauna database to somewhere else.
 * How to use it:
 * 1. Fill out the COLLECTION_NAMES array with the names of the collections you want to migrate
 * 2. Fill out the migrateDocument function with the logic to migrate a single document
 * 3. Make sure your FAUNA_SECRET and FAUNA_ENDPOINT variables are set up correctly
 * 4. Run the script with `npx tsx migrate-fauna-data.ts`
 *
 * Please add feedback or suggestions in the comments!
 */

/* eslint-disable no-console */
import { fql, Client, Page } from 'fauna'

const COLLECTION_NAMES = [
	`User`,
	// ...etc
] as const

const PAGE_SIZE = 10

const client = new Client({
	secret: process.env.FAUNA_SECRET ?? '',
	endpoint: new URL(process.env.FAUNA_ENDPOINT ?? ''),
})

const migrateDocument = async (
	doc: any,
	collectionName: typeof COLLECTION_NAMES[number]
) => {
	// TODO: Place your code to migrate the document here
}

const migrateData = async () => {
	for await (const collectionName of COLLECTION_NAMES) {
		try {
			console.log(`⛳️ Starting migration of ${collectionName}`)

			// Get the first page of documents
			const result = await client.query<{
				firstPage: Page<any>
				count: number
			}>(
				fql`
				let all = Collection(${collectionName}).all()
				let count = all.count()
				{
					firstPage: all.paginate(${PAGE_SIZE}),
					count: count,
				}
				`
			)
			await migrateCollectionDocuments(
				result.data.firstPage,
				0,
				result.data.count,
				collectionName
			)
		} catch (error) {
			console.error(`❌ Error migrating ${collectionName}`)
			console.error(error)
			break
		}
	}
}

const migrateCollectionDocuments = async (
	page: Page<any>,
	pageNumber: number,
	count: number,
	collectionName: typeof COLLECTION_NAMES[number]
) => {
	const from = count === 0 ? 0 : pageNumber * PAGE_SIZE + 1
	let to = pageNumber * PAGE_SIZE + PAGE_SIZE
	if (to > count) {
		to = count
	}

	console.log(`🔄 Migrating ${collectionName} (${from}-${to}/${count})`)
	for await (const document of page.data) {
		try {
			await migrateDocument(document, collectionName)
		} catch (error) {
			console.log(`Failed migrating document: ${JSON.stringify(document)}`)
			throw new Error(JSON.stringify(error))
		}
	}

	if (page.after) {
		// Get the next page of documents
		const nextPageResult = await client.query<Page<any>>(
			fql`Set.paginate(${page.after})`
		)
		await migrateCollectionDocuments(
			nextPageResult.data,
			pageNumber + 1,
			count,
			collectionName
		)
	} else {
		console.log(`✅ ${collectionName} has been migrated`)
	}
}

migrateData()

Possible improvements:

• Retry logic

Potential issues:

• If an error occurs in the process of migrating, there's no way to roll back the previous changes made during the migration (no staging of the migration) – this could be helped by logging the page cursor and index at which the error happened and accepting them as inputs into the script (to continue from this point onwards)