Find missing assets in dataset

find-missing-assets.ts

import got from "got"
import createClient from "@sanity/client"
import {asyncScheduler, concat, defer, EMPTY, from} from "rxjs"
import {
  map,
  mergeMap,
  retry,
  scan,
  takeLast,
  tap,
  throttleTime,
  timeout,
} from "rxjs/operators"
import {uniq} from "./uniq"

const BATCH_SIZE = 1000
const START_AT = 0

const PROJECT_ID = '<projectId>'
const DATASET = '<dataset>'

const client = createClient({
  projectId: PROJECT_ID,
  dataset: DATASET,
  useCdn: true,
})

const checkMissingImage = (url) =>
  defer(() => got.head(url, {throwHttpErrors: false})).pipe(
    timeout(10000),
    map((response) => response.statusCode),
    tap((statusCode) => {
      if (statusCode !== 200 && statusCode !== 404) {
        throw new Error(`Unexpected status code ${statusCode} from HEAD ${url}`)
      }
    }),
    retry(4),
    map((statusCode) => statusCode === 200)
  )

interface BatchProgress {
  batch: [number, number]
  remainingInBatch: number
  missingUrls: string[]
}

const getNextBatch = (start, end) =>
  defer(() =>
    client.observable.fetch(
      `*[_type == "sanity.imageAsset"] | order(_createdAt asc) [${start}...${end}].url`
    )
  ).pipe(
    mergeMap((imageUrls) => {
      if (imageUrls.length === 0) {
        return EMPTY
      }
      return concat(
        from(imageUrls).pipe(
          mergeMap(
            (url) =>
              checkMissingImage(url).pipe(
                map((exists) => (exists ? null : url))
              ),
            500
          ),
          scan(
            (acc: BatchProgress, missingUrl: string | null): BatchProgress => ({
              ...acc,
              missingUrls: acc.missingUrls.concat(missingUrl ? missingUrl : []),
              remainingInBatch: acc.remainingInBatch - 1,
            }),
            {
              batch: [start, end],
              remainingInBatch: imageUrls.length,
              missingUrls: [],
            }
          )
        ),
        getNextBatch(end, end + BATCH_SIZE)
      )
    })
  )

const run$ = getNextBatch(START_AT, START_AT + BATCH_SIZE).pipe(
  scan(
    (acc: BatchProgress | null, nextState: BatchProgress): BatchProgress => ({
      ...nextState,
      missingUrls: acc
        ? uniq(acc.missingUrls.concat(nextState.missingUrls))
        : [],
    })
  ),
  throttleTime(500, asyncScheduler, {leading: true, trailing: true}),
  tap((state: BatchProgress) => {
    console.clear()
    console.log(
      "Current batch: %d…%d (%d remaining in batch)",
      state.batch[0],
      state.batch[1],
      state.remainingInBatch
    )
    console.log(
      "Found %d missing image(s) so far\n%s",
      state.missingUrls.length,
      state.missingUrls.join("\n")
    )
  }),
  takeLast(1),
  map((final: BatchProgress) => {
    console.clear()
    console.log(
      "Done! Found %d missing images\n%s",
      final.missingUrls.length,
      final.missingUrls.join("\n")
    )
  })
)

run$.subscribe()