Downloading tree-sitter WASM modules needed by OpenTUI

create-opentui-cache-bundle.ts

#!/usr/bin/env bun

import { existsSync } from "node:fs"
import path from "node:path"
import { mkdir, readFile, rm, writeFile } from "node:fs/promises"
import { pathToFileURL } from "node:url"

type Tier = "recommended" | "second" | "niche"

type ParserDefinition = {
  filetype: string
  tier: Tier
  wasm: string
  highlights: string[]
  injections: string[]
}

type DownloadJob = {
  source: string
  destination: string
}

type ParserConfigModule = {
  parsers?: {
    filetype?: string
    wasm?: string
    queries?: {
      highlights?: string[]
      injections?: string[]
    }
  }[]
}

const DEFAULT_CONFIG_PATH = path.resolve(import.meta.dir, "parsers-config.ts")
const TIERS: readonly Tier[] = ["recommended", "second", "niche"]
const RECOMMENDED_FILETYPES = new Set([
  "bash",
  "python",
  "json",
  "yaml",
  "toml",
  "diff",
  "go",
  "rust",
  "c",
  "cpp",
  "java",
  "html",
  "css",
  "nix",
  "javascript",
  "typescript",
  "markdown",
  "markdown_inline",
  "zig",
])
const SECOND_FILETYPES = new Set([
  "csharp",
  "kotlin",
  "php",
  "ruby",
  "scala",
  "vue",
  "hcl",
  "lua",
  "swift",
  "elixir",
  "powershell",
])

const usage = [
  "Usage: bun ./create-opentui-cache-bundle.ts [--config <path>] [--out <dir>] [--archive <file>] [--no-archive] [--tiers <tiers>] [--concurrency <n>] [--force] [--list]",
  "",
  "Downloads OpenTUI-compatible tree-sitter cache files to:",
  "  <out>/tree-sitter/languages",
  "  <out>/tree-sitter/queries",
  "",
  "Options:",
  "  --config <path>      Parser config module to load (default: auto-detect, then ./parsers-config.ts)",
  "  --out <dir>          Output root directory (default: ./opentui-cache)",
  "  --archive <file>     Output tar.gz archive path (default: <out>.tar.gz)",
  "  --no-archive         Skip creating the tar.gz archive",
  "  --tiers <list>       Comma-separated tiers: recommended,second,niche,all (default: all)",
  "  --concurrency <n>    Number of concurrent downloads (default: 8)",
  "  --force              Re-download even if cache file already exists",
  "  --list               Print the parser list and exit",
  "  -h, --help           Show this help",
  "",
  "Notes:",
  "  - By default, the script tries to detect an installed @opencode-ai/tui parsers-config.ts, then falls back to the local vendored copy.",
  "  - You can point --config at OpenTUI's parser config to mirror its bundled asset list instead.",
  "  - Cache file names mirror OpenTUI's tree-sitter DownloadUtils naming.",
].join("\n")

type Options = {
  configPath: string
  configSource: "explicit" | "detected" | "vendored"
  outDir: string
  archivePath: string | null
  tiers: Set<Tier>
  concurrency: number
  force: boolean
  list: boolean
}

type ResolvedConfig = Pick<Options, "configPath" | "configSource">

const options = parseArgs(process.argv.slice(2))
const parsers = await loadParsers(options.configPath)

if (options.list) {
  printList(parsers, options.tiers)
  process.exit(0)
}

const selected = parsers.filter((parser) => options.tiers.has(parser.tier))
const rootDir = path.resolve(options.outDir)
const languagesDir = path.join(rootDir, "tree-sitter", "languages")
const queriesDir = path.join(rootDir, "tree-sitter", "queries")

await mkdir(languagesDir, { recursive: true })
await mkdir(queriesDir, { recursive: true })

console.log(`Config: ${path.resolve(options.configPath)}`)
console.log(`Config source: ${options.configSource}`)
console.log(`Output root: ${rootDir}`)
console.log(`Downloading ${selected.length} parser definitions across tiers: ${Array.from(options.tiers).join(", ")}`)
console.log(`Using concurrency: ${options.concurrency}`)
if (options.archivePath) {
  console.log(`Archive path: ${path.resolve(options.archivePath)}`)
}

let downloaded = 0
let skipped = 0

const jobs = buildJobs(selected, languagesDir, queriesDir)
console.log(`Queued ${jobs.length} files`)

for (const result of await runWithConcurrency(jobs, options.concurrency, (job) => fetchInto(job, options.force))) {
  count(result)
}

console.log(`\nDone.`)
console.log(`Downloaded: ${downloaded}`)
console.log(`Skipped: ${skipped}`)

if (options.archivePath) {
  const archivePath = path.resolve(options.archivePath)
  await createArchive(rootDir, archivePath)
  console.log(`Archive: ${archivePath}`)
}

function parseArgs(args: string[]): Options {
  if (args.includes("--help") || args.includes("-h")) {
    console.log(usage)
    process.exit(0)
  }

  let explicitConfigPath: string | null = null
  let outDir = "./opentui-cache"
  let archivePath: string | null = null
  let archiveEnabled = true
  let tierArg = "all"
  let concurrency = 8
  let force = false
  let list = false

  for (let index = 0; index < args.length; index++) {
    const arg = args[index]
    if (arg === "--config") {
      const value = args[index + 1]
      if (!value) fail("Missing value for --config")
      explicitConfigPath = value
      index += 1
      continue
    }
    if (arg.startsWith("--config=")) {
      explicitConfigPath = arg.slice("--config=".length)
      continue
    }
    if (arg === "--out") {
      const value = args[index + 1]
      if (!value) fail("Missing value for --out")
      outDir = value
      index += 1
      continue
    }
    if (arg.startsWith("--out=")) {
      outDir = arg.slice("--out=".length)
      continue
    }
    if (arg === "--archive") {
      const value = args[index + 1]
      if (!value) fail("Missing value for --archive")
      archivePath = value
      archiveEnabled = true
      index += 1
      continue
    }
    if (arg.startsWith("--archive=")) {
      archivePath = arg.slice("--archive=".length)
      archiveEnabled = true
      continue
    }
    if (arg === "--no-archive") {
      archiveEnabled = false
      archivePath = null
      continue
    }
    if (arg === "--tiers") {
      const value = args[index + 1]
      if (!value) fail("Missing value for --tiers")
      tierArg = value
      index += 1
      continue
    }
    if (arg.startsWith("--tiers=")) {
      tierArg = arg.slice("--tiers=".length)
      continue
    }
    if (arg === "--force") {
      force = true
      continue
    }
    if (arg === "--concurrency") {
      const value = args[index + 1]
      if (!value) fail("Missing value for --concurrency")
      concurrency = parsePositiveInteger(value, "--concurrency")
      index += 1
      continue
    }
    if (arg.startsWith("--concurrency=")) {
      concurrency = parsePositiveInteger(arg.slice("--concurrency=".length), "--concurrency")
      continue
    }
    if (arg === "--list") {
      list = true
      continue
    }
    fail(`Unknown option: ${arg}`)
  }

  const tiers = parseTiers(tierArg)
  const resolvedConfig = resolveConfig(explicitConfigPath)
  return {
    ...resolvedConfig,
    outDir,
    archivePath: archiveEnabled ? archivePath ?? `${outDir}.tar.gz` : null,
    tiers,
    concurrency,
    force,
    list,
  }
}

function resolveConfig(explicitConfigPath: string | null): ResolvedConfig {
  if (explicitConfigPath) {
    return { configPath: path.resolve(explicitConfigPath), configSource: "explicit" }
  }

  const detectedConfigPath = resolveInstalledOpencodeConfig()
  if (detectedConfigPath) {
    return { configPath: detectedConfigPath, configSource: "detected" }
  }

  return { configPath: DEFAULT_CONFIG_PATH, configSource: "vendored" }
}

function resolveInstalledOpencodeConfig() {
  const fromCwd = findInstalledConfig(process.cwd())
  if (fromCwd) return fromCwd

  const opencodeBinary = Bun.which("opencode")
  if (!opencodeBinary) return null

  return findInstalledConfig(path.dirname(opencodeBinary))
}

function findInstalledConfig(startDir: string) {
  let current = path.resolve(startDir)

  while (true) {
    const candidate = path.join(current, "node_modules", "@opencode-ai", "tui", "src", "parsers-config.ts")
    if (existsSync(candidate)) {
      return candidate
    }

    const parent = path.dirname(current)
    if (parent === current) {
      return null
    }
    current = parent
  }
}

function parseTiers(input: string) {
  if (input === "all") return new Set(TIERS)

  const values = input
    .split(",")
    .map((value) => value.trim())
    .filter((value): value is Tier => TIERS.includes(value as Tier))

  if (values.length === 0) fail(`Invalid --tiers value: ${input}`)
  return new Set(values)
}

function printList(parsers: ParserDefinition[], tiers: Set<Tier>) {
  for (const tier of TIERS) {
    if (!tiers.has(tier)) continue
    console.log(`${tier}:`)
    for (const parser of parsers.filter((item) => item.tier === tier)) {
      console.log(`- ${parser.filetype}`)
    }
    console.log("")
  }
}

function fail(message: string): never {
  console.error(message)
  console.error(usage)
  process.exit(1)
}

function parsePositiveInteger(input: string, flag: string) {
  const value = Number.parseInt(input, 10)
  if (!Number.isInteger(value) || value < 1) {
    fail(`Invalid value for ${flag}: ${input}`)
  }
  return value
}

async function loadParsers(configPath: string) {
  let imported: { default?: ParserConfigModule }
  try {
    imported = await import(pathToFileURL(configPath).href)
  } catch (error) {
    fail(`Failed to load parser config ${configPath}: ${error}`)
  }

  const parsers = imported.default?.parsers
  if (!parsers || !Array.isArray(parsers)) {
    fail(`Parser config ${configPath} does not export a default { parsers: [...] } object`)
  }

  return parsers.map((parser, index) => {
    if (!parser?.filetype || !parser.wasm || !parser.queries?.highlights) {
      fail(`Invalid parser entry at index ${index} in ${configPath}`)
    }

    return {
      filetype: parser.filetype,
      tier: classifyParser(parser.filetype),
      wasm: resolveSource(configPath, parser.wasm),
      highlights: parser.queries.highlights.map((source) => resolveSource(configPath, source)),
      injections: (parser.queries.injections ?? []).map((source) => resolveSource(configPath, source)),
    }
  })
}

function classifyParser(filetype: string): Tier {
  if (RECOMMENDED_FILETYPES.has(filetype)) return "recommended"
  if (SECOND_FILETYPES.has(filetype)) return "second"
  return "niche"
}

function resolveSource(configPath: string, source: string) {
  if (isUrl(source) || path.isAbsolute(source)) return source
  return path.resolve(path.dirname(configPath), source)
}

function isUrl(source: string) {
  return source.startsWith("http://") || source.startsWith("https://")
}

function hashSource(source: string) {
  let hash = 0
  for (let index = 0; index < source.length; index++) {
    const char = source.charCodeAt(index)
    hash = (hash << 5) - hash + char
    hash &= hash
  }
  return Math.abs(hash).toString(16)
}

function sourceBasename(source: string) {
  if (isUrl(source)) {
    return path.basename(new URL(source).pathname)
  }

  return path.basename(source)
}

function buildJobs(selected: ParserDefinition[], languagesDir: string, queriesDir: string) {
  const jobs: DownloadJob[] = []

  for (const parser of selected) {
    jobs.push({
      source: parser.wasm,
      destination: path.join(languagesDir, sourceBasename(parser.wasm)),
    })

    for (const source of [...parser.highlights, ...parser.injections]) {
      jobs.push({
        source,
        destination: path.join(queriesDir, `${parser.filetype}-${hashSource(source)}.scm`),
      })
    }
  }

  return jobs
}

async function runWithConcurrency<T, R>(items: T[], concurrency: number, worker: (item: T) => Promise<R>) {
  if (items.length === 0) return [] as R[]

  const results = new Array<R>(items.length)
  let nextIndex = 0

  async function runWorker() {
    while (true) {
      const index = nextIndex
      nextIndex += 1
      if (index >= items.length) return
      results[index] = await worker(items[index])
    }
  }

  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runWorker())
  await Promise.all(workers)
  return results
}

async function fetchInto(job: DownloadJob, force: boolean) {
  const { source, destination } = job

  if (!force) {
    try {
      const existing = await readFile(destination)
      if (existing.byteLength > 0) {
        console.log(`  cached ${path.relative(rootDir, destination)}`)
        return false
      }
    } catch {
      // cache miss
    }
  }

  console.log(`  ${isUrl(source) ? "fetch" : "copy "} ${path.relative(rootDir, destination)}`)

  const bytes = isUrl(source) ? await fetchBytes(source) : await readFile(source)
  await mkdir(path.dirname(destination), { recursive: true })
  await writeFile(destination, bytes)
  console.log(`  wrote  ${path.relative(rootDir, destination)}`)
  return true
}

async function fetchBytes(source: string) {
  const response = await fetch(source)
  if (!response.ok) {
    throw new Error(`Failed to fetch ${source}: ${response.status} ${response.statusText}`)
  }

  return Buffer.from(await response.arrayBuffer())
}

function count(result: boolean) {
  if (result) downloaded += 1
  else skipped += 1
}

async function createArchive(rootDir: string, archivePath: string) {
  await rm(archivePath, { force: true })

  const proc = Bun.spawn(["tar", "-czf", archivePath, path.basename(rootDir)], {
    cwd: path.dirname(rootDir),
    stdout: "inherit",
    stderr: "inherit",
  })

  const exitCode = await proc.exited
  if (exitCode !== 0) {
    throw new Error(`tar failed with exit code ${exitCode}`)
  }
}