Run bun test with minimal output

bun-test-with-minimal-output.md

bun test is really fast, but the output is verbose, which makes it less than ideal for using in a TDD cycle with an LLM.

This hack runs bun test with a custom reporter to reduce noise down to bare minimum.

It's a hack because at this time, the events emitted by bun are missing a few bits that would make it easy.

The output is as minimal as possible:

> bun bun-test-with-minimal-output.ts

[FAIL] Math > fails on purpose

Error: expect(received).toBe(expected)

Expected: 3
Received: 2

  at example.test.ts:18:19

13 |   it.skip('divides numbers', () => {
14 |     expect(10 / 2).toBe(5)
15 |   })
16 | 
17 |   it('fails on purpose', () => {
18 |     expect(1 + 1).toBe(3)
                      ^

----------------

[FAIL] Math > fails again on purpose

Error: This function always throws an error
  at example.test.include.ts:3:11
  at example.test.ts:30:9

1 | export function throwError() {
2 |     // This function is designed to always throw an error
3 |     throw new Error('This function always throws an error');
              ^

----------------

PASS: 3
FAIL: 2

The code is largely borrowed from here

bun-test-with-minimal-output.ts

import { tmpdir } from 'node:os'
import path from 'node:path'

interface Socket<T = unknown> {
  data: T
  write(data: string | Buffer): void
  unref(): void
  ref(): void
}

enum FramerState {
  WaitingForLength = 0,
  WaitingForMessage = 1,
}

let socketFramerMessageLengthBuffer: Buffer
export class SocketFramer {
  private state: FramerState = FramerState.WaitingForLength
  private pendingLength = 0
  private sizeBuffer: Buffer = Buffer.alloc(0)
  private sizeBufferIndex = 0
  private bufferedData: Buffer = Buffer.alloc(0)

  constructor(private onMessage: (message: string) => void) {
    if (!socketFramerMessageLengthBuffer) {
      socketFramerMessageLengthBuffer = Buffer.alloc(4)
    }
    this.reset()
  }

  reset(): void {
    this.state = FramerState.WaitingForLength
    this.bufferedData = Buffer.alloc(0)
    this.sizeBufferIndex = 0
    this.sizeBuffer = Buffer.alloc(4)
  }

  send(socket: Socket, data: string): void {
    socketFramerMessageLengthBuffer.writeUInt32BE(Buffer.byteLength(data), 0)
    socket.write(socketFramerMessageLengthBuffer)
    socket.write(data)
  }

  onData(_socket: Socket, data: Buffer): void {
    this.bufferedData =
      this.bufferedData.length > 0
        ? Buffer.concat([this.bufferedData, data])
        : data

    const messagesToDeliver: string[] = []

    while (this.bufferedData.length > 0) {
      if (this.state === FramerState.WaitingForLength) {
        if (this.sizeBufferIndex + this.bufferedData.length < 4) {
          const remainingBytes = Math.min(
            4 - this.sizeBufferIndex,
            this.bufferedData.length
          )
          this.bufferedData.copy(
            this.sizeBuffer,
            this.sizeBufferIndex,
            0,
            remainingBytes
          )
          this.sizeBufferIndex += remainingBytes
          this.bufferedData = this.bufferedData.slice(remainingBytes)
          break
        }

        const remainingBytes = 4 - this.sizeBufferIndex
        this.bufferedData.copy(
          this.sizeBuffer,
          this.sizeBufferIndex,
          0,
          remainingBytes
        )
        this.pendingLength = this.sizeBuffer.readUInt32BE(0)

        this.state = FramerState.WaitingForMessage
        this.sizeBufferIndex = 0
        this.bufferedData = this.bufferedData.slice(remainingBytes)
      }

      if (this.bufferedData.length < this.pendingLength) {
        break
      }

      const message = this.bufferedData.toString('utf-8', 0, this.pendingLength)
      this.bufferedData = this.bufferedData.slice(this.pendingLength)
      this.state = FramerState.WaitingForLength
      this.pendingLength = 0
      this.sizeBufferIndex = 0
      messagesToDeliver.push(message)
    }

    for (const message of messagesToDeliver) {
      this.onMessage(message)
    }
  }
}

function randomUnixPath(): string {
  return path.join(tmpdir(), `${Math.random().toString(36).slice(2)}.sock`)
}

const DOMAINS = [
  'Console',
  'Inspector',
  'LifecycleReporter',
  'Runtime',
  'TestReporter',
]

class Inspector {
  protected messageCallbacks: Map<number, (result: unknown) => void>
  protected eventListeners: Map<string, ((params: unknown) => void)[]>
  nextId: number
  framer?: SocketFramer
  socket?: Socket<{ onData: (socket: Socket<unknown>, data: Buffer) => void }>

  constructor() {
    this.messageCallbacks = new Map()
    this.eventListeners = new Map()
    this.nextId = 1
  }

  onMessage(data: string) {
    // console.log("Received message:", data);
    try {
      const message = JSON.parse(data)
      if (message.id && this.messageCallbacks.has(message.id)) {
        const callback = this.messageCallbacks.get(message.id)
        if (callback) {
          this.messageCallbacks.delete(message.id)
          callback(message.result)
        }
      } else if (message.method) {
        const listeners = this.eventListeners.get(message.method)
        if (listeners) {
          for (const listener of listeners) {
            listener(message.params)
          }
        }
      }
    } catch (error) {
      console.error('Error parsing message:', error)
      return
    }
  }

  send(method: string, params: unknown = {}) {
    if (!this.framer) throw new Error('Socket not connected')
    const id = this.nextId++
    const message = { id, method, params }
    this.framer.send(
      this.socket as Socket<{
        onData: (socket: Socket<unknown>, data: Buffer) => void
      }>,
      JSON.stringify(message)
    )
  }

  addEventListener(method: string, callback: (params: unknown) => void) {
    if (!this.eventListeners.has(method)) {
      this.eventListeners.set(method, [])
    }
    const listeners = this.eventListeners.get(method)
    if (listeners) {
      listeners.push(callback)
    }
  }

  enable() {
    for (const domain of DOMAINS) {
      this.send(`${domain}.enable`)
    }
  }

  initialize() {
    this.send('Inspector.initialized')
  }

  unref() {
    this.socket?.unref()
  }

  ref() {
    this.socket?.ref()
  }
}

async function connect(
  address: string
): Promise<
  Socket<{ onData: (socket: Socket<unknown>, data: Buffer) => void }>
> {
  const { promise, resolve } =
    Promise.withResolvers<
      Socket<{ onData: (socket: Socket<unknown>, data: Buffer) => void }>
    >()

  const listener = Bun.listen<{
    onData: (socket: Socket<unknown>, data: Buffer) => void
  }>({
    unix: address.slice('unix://'.length),
    socket: {
      open: socket => {
        listener.stop()
        socket.ref()
        resolve(socket)
      },
      data(socket, data: Buffer) {
        socket.data?.onData(socket, data)
      },
      error(_socket, error) {
        console.error(error)
      },
      close(_socket) {},
    },
  })

  return await promise
}

const url = `unix://${randomUnixPath()}`
const socketPromise = connect(url)
const proc = Bun.spawn({
  cmd: [
    process.execPath,
    `--inspect-wait=${url}`,
    'test',
    ...process.argv.slice(2),
  ],
  env: {
    ...process.env,
  },
  stdout: 'pipe',
  stderr: 'pipe',
})

const stdoutChunks: Uint8Array[] = []
const stderrChunks: Uint8Array[] = []

const stdoutReader = proc.stdout?.getReader()
const stderrReader = proc.stderr?.getReader()

async function readStream(
  reader: NonNullable<typeof stdoutReader>,
  chunks: Uint8Array[]
) {
  if (!reader) return
  try {
    while (true) {
      const { done, value } = await reader.read()
      if (done) break
      chunks.push(value)
    }
  } catch (error) {
    console.error('Error reading stream:', error)
  }
}

type TestError = {
  message: string
  name: string
  urls: string[]
  lineColumns: number[]
  sourceLines: string[]
}

type Test = {
  name: string
  status: 'discovered' | 'started' | 'pass' | 'skip' | 'todo' | 'fail'
  duration?: number
  error?: TestError
}

const tests: Map<number, Test> = new Map()
const stats = {
  pass: 0,
  fail: 0,
}

if (stdoutReader && stderrReader) {
  const stdoutPromise = readStream(stdoutReader, stdoutChunks)
  const stderrPromise = readStream(stderrReader, stderrChunks)

  proc.exited.then(
    async _exitCode => {
      await Promise.all([stdoutPromise, stderrPromise])

      const stderr = Buffer.concat(stderrChunks)

      const testNames = stderr
        .toString()
        .split('\n')
        .map(line => line.match(/\((pass|fail)\)\s(.+?)(\s\[\d+\.\d+ms\])?$/))
        .map(match => (match ? match[2] : null))
        .filter(Boolean)

      const testsArray = Array.from(tests.values()).map((test, index) => ({
        ...test,
        fullName: testNames[index] || test.name,
      }))

      for (const test of testsArray) {
        if (test.status === 'fail') {
          process.stderr.write(`[FAIL] ${test.fullName}\n\n`)
          if (test.error) {
            process.stderr.write(`${test.error.name}: ${test.error.message}\n`)
            for (let i = 0; i < test.error.urls.length; i++) {
              const url = test.error.urls[i]
              if (url) {
                process.stderr.write(
                  `  at ${cwdRelativePath(url)}:${test.error.lineColumns[i * 2]}:${test.error.lineColumns[i * 2 + 1]}\n`
                )
              }
            }
            process.stderr.write('\n')

            const [lineNumber, columnNumber] = test.error.lineColumns as [
              number,
              number,
            ]

            const sourceListing = [
              ...test.error.sourceLines
                .map((line, index) => {
                  return `${lineNumber - index} | ${line.replace(/^\n/, '').replace(/\n$/, '')}`
                })
                .reverse(),
              `${' '.repeat(columnNumber)}   ^`,
            ]

            process.stderr.write(
              `${sourceListing.join('\n')}\n\n----------------\n\n`
            )
          }
        }
      }

      process.stderr.write(`PASS: ${stats.pass}\nFAIL: ${stats.fail}\n`)

      process.exit(stats.fail)
    },
    error => {
      console.error(error)
      process.exit(1)
    }
  )

  const inspector = new Inspector()

  inspector.addEventListener('TestReporter.found', params => {
    const typedParams = params as { id: number; name: string }
    const test: Test = {
      name: typedParams.name,
      status: 'discovered',
    }
    tests.set(typedParams.id, test)
  })

  let currentTestId: number | null = null

  inspector.addEventListener('TestReporter.start', params => {
    const typedParams = params as { id: number }
    currentTestId = typedParams.id
    const test = tests.get(typedParams.id)
    if (test) {
      test.status = 'started'
    }
  })

  inspector.addEventListener('TestReporter.end', params => {
    const typedParams = params as {
      id: number
      elapsed: number
      status: 'pass' | 'skip' | 'todo' | 'fail'
    }
    const test = tests.get(typedParams.id)
    if (test) {
      test.status = typedParams.status
      test.duration = typedParams.elapsed
    }
    if (typedParams.status === 'pass' || typedParams.status === 'fail') {
      stats[typedParams.status]++
    }
  })

  inspector.addEventListener('LifecycleReporter.error', params => {
    const typedParams = params as TestError
    if (currentTestId !== null) {
      const test = tests.get(currentTestId)
      if (test) {
        test.error = {
          message: typedParams.message,
          name: typedParams.name,
          urls: typedParams.urls,
          lineColumns: typedParams.lineColumns,
          sourceLines: typedParams.sourceLines,
        }
      }
    }
  })

  const socket = await socketPromise
  const framer = new SocketFramer((message: string) => {
    inspector.onMessage(message)
  })
  inspector.socket = socket
  inspector.framer = framer
  socket.data = {
    onData: framer.onData.bind(framer),
  }

  inspector.enable()
  inspector.initialize()
  inspector.unref()
}

function cwdRelativePath(filePath: string): string {
  const cwd = process.cwd()
  if (filePath.startsWith(cwd)) {
    return filePath.slice(cwd.length + 1)
  }
  return filePath
}