zig test runner

test_runner.zig

// Modded from from https://gist.github.com/karlseguin/c6bea5b35e4e8d26af6f81c22cb5d76b

// in your build.zig, you can specify a custom test runner:
// const tests = b.addTest(.{
//   .target = target,
//   .optimize = optimize,
//   .test_runner = .{ .path = b.path("test_runner.zig"), .mode = .simple }, // add this line
//   .root_source_file = b.path("src/main.zig"),
// });
//
// Tested on versions:
// - zig-0.14
// - zig-0.15.0-dev.1092+d772c0627.
//
// Note: doesn't support std.testing.fuzz()

const std = @import("std");
const builtin = @import("builtin");

const BORDER = "=" ** 80;

const Status = enum {
    pass,
    fail,
    skip,
    text,
};

fn getenvOwned(alloc: std.mem.Allocator, key: []const u8) ?[]u8 {
    const v = std.process.getEnvVarOwned(alloc, key) catch |err| {
        if (err == error.EnvironmentVariableNotFound) {
            return null;
        }
        std.log.warn("failed to get env var {s} due to err {}", .{ key, err });
        return null;
    };
    return v;
}

pub fn main() !void {
    var gpa = std.heap.GeneralPurposeAllocator(.{ .stack_trace_frames = 12 }){};
    const alloc = gpa.allocator();
    const fail_first = blk: {
        if (getenvOwned(alloc, "TEST_FAIL_FIRST")) |e| {
            defer alloc.free(e);
            break :blk std.mem.eql(u8, e, "true");
        }
        break :blk false;
    };
    const filter = getenvOwned(alloc, "TEST_FILTER");
    defer if (filter) |f| alloc.free(f);

    const printer = Printer.init();
    printer.fmt("\r\x1b[0K", .{}); // beginning of line and clear to end of line

    var pass: usize = 0;
    var fail: usize = 0;
    var skip: usize = 0;
    var leak: usize = 0;

    for (builtin.test_functions) |t| {
        std.testing.allocator_instance = .{};
        var status = Status.pass;

        if (filter) |f| {
            if (std.mem.indexOf(u8, t.name, f) == null) {
                continue;
            }
        }

        printer.fmt("Testing {s}: ", .{t.name});
        const result = t.func();

        if (std.testing.allocator_instance.deinit() == .leak) {
            leak += 1;
            printer.status(.fail, "\n{s}\n\"{s}\" - Memory Leak\n{s}\n", .{ BORDER, t.name, BORDER });
        }

        if (result) |_| {
            pass += 1;
        } else |err| {
            switch (err) {
                error.SkipZigTest => {
                    skip += 1;
                    status = .skip;
                },
                else => {
                    status = .fail;
                    fail += 1;
                    printer.status(.fail, "\n{s}\n\"{s}\" - {s}\n{s}\n", .{ BORDER, t.name, @errorName(err), BORDER });
                    if (@errorReturnTrace()) |trace| {
                        std.debug.dumpStackTrace(trace.*);
                    }
                    if (fail_first) {
                        break;
                    }
                },
            }
        }

        printer.status(status, "[{s}]\n", .{@tagName(status)});
    }

    const total_tests = pass + fail;
    const status: Status = if (fail == 0) .pass else .fail;
    printer.status(status, "\n{d} of {d} test{s} passed\n", .{ pass, total_tests, if (total_tests != 1) "s" else "" });
    if (skip > 0) {
        printer.status(.skip, "{d} test{s} skipped\n", .{ skip, if (skip != 1) "s" else "" });
    }
    if (leak > 0) {
        printer.status(.fail, "{d} test{s} leaked\n", .{ leak, if (leak != 1) "s" else "" });
    }
    std.process.exit(if (fail == 0) 0 else 1);
}

const Printer = struct {
    fn init() Printer {
        return .{};
    }

    fn fmt(self: Printer, comptime format: []const u8, args: anytype) void {
        _ = self; // autofix
        std.debug.print(format, args);
    }

    fn status(self: Printer, s: Status, comptime format: []const u8, args: anytype) void {
        const color = switch (s) {
            .pass => "\x1b[32m",
            .fail => "\x1b[31m",
            .skip => "\x1b[33m",
            else => "",
        };
        std.debug.print("{s}", .{color});
        std.debug.print(format, args);
        self.fmt("\x1b[0m", .{});
    }
};

Thanks @mamahnxarya201, your change is fine. I updated the gist content. It should work with zig-0.14 now.

This no longer works for a recent zig master version (mine: 0.15.0-dev.1034+bd97b6618). My adapted, ported and working version is here: https://github.com/utensil/native-land/blob/main/yard-zig/basic-xp/test_runner.zig

It also added some features and tweaks, before the port. It looks like this for zig build test --summary all:

// omitted....
simd.test.vector swizzle operations [pass]
simd.test.vector min/max operations [pass]
simd.test.vector bitwise operations [pass]
simd.test.vector floating point operations [pass]
simd.test.vector load and store operations [pass]
simd: 12 of 12 tests passed
string_literals.test.multiline string literals vs regular strings [pass]
string_literals.test.multiline strings preserve backslashes [pass]
string_literals.test.multiline strings for code generation [pass]
string_literals.test.multiline strings preserve trailing whitespace [pass]
string_literals.test.@embedFile vs multiline strings [pass]
string_literals: 5 of 5 tests passed
Total: 83 of 83 tests passed
Build Summary: 5/5 steps succeeded
test success
+- run test success 243ms MaxRSS:2M
|  +- compile test Debug native success 842ms MaxRSS:282M
+- run test success 239ms MaxRSS:1M
   +- compile test Debug native success 719ms MaxRSS:244M

Disclaimer: Some code is coded with the help of a coding agent. It figured out how to fix the errors by trying a few different ways. I checked the latest document and nudged it in the right direction.

Thanks @utensil! I updated to gist to use std.debug.print() (like in your version) and it should now work on zig-0.14 and 0.15.0-dev.1092+d772c0627. Using std.debug.print is probably a safer path to keep this compiling than updating to use the new Io interface. (Honestly, I couldn't get the old std.fs.File based code to compile. ;))