zig test runner

test_runner.zig

// Modded from from https://gist.github.com/karlseguin/c6bea5b35e4e8d26af6f81c22cb5d76b

// in your build.zig, you can specify a custom test runner:
// const tests = b.addTest(.{
//   .target = target,
//   .optimize = optimize,
//   .test_runner = .{ .path = b.path("test_runner.zig"), .mode = .simple }, // add this line
//   .root_source_file = b.path("src/main.zig"),
// });
//
// Tested on versions:
// - zig-0.16.0
//
// Note: doesn't support std.testing.fuzz()

const std = @import("std");
const builtin = @import("builtin");

const BORDER = "=" ** 80;

const Status = enum {
    pass,
    fail,
    skip,
    text,
};

pub fn main(process_init: std.process.Init) !void {
    const fail_first = blk: {
        if (process_init.environ_map.get("TEST_FAIL_FIRST")) |e| {
            break :blk std.mem.eql(u8, e, "true");
        }
        break :blk false;
    };
    const filter = process_init.environ_map.get("TEST_FILTER");

    const printer = Printer.init();
    printer.fmt("\r\x1b[0K", .{}); // beginning of line and clear to end of line

    var pass: usize = 0;
    var fail: usize = 0;
    var skip: usize = 0;
    var leak: usize = 0;

    for (builtin.test_functions) |t| {
        std.testing.allocator_instance = .{};
        var status = Status.pass;

        if (filter) |f| {
            if (std.mem.indexOf(u8, t.name, f) == null) {
                continue;
            }
        }

        printer.fmt("Testing {s}: ", .{t.name});
        const result = t.func();

        if (std.testing.allocator_instance.deinit() == .leak) {
            leak += 1;
            printer.status(.fail, "\n{s}\n\"{s}\" - Memory Leak\n{s}\n", .{ BORDER, t.name, BORDER });
        }

        if (result) |_| {
            pass += 1;
        } else |err| {
            switch (err) {
                error.SkipZigTest => {
                    skip += 1;
                    status = .skip;
                },
                else => {
                    status = .fail;
                    fail += 1;
                    printer.status(.fail, "\n{s}\n\"{s}\" - {s}\n{s}\n", .{ BORDER, t.name, @errorName(err), BORDER });
                    if (@errorReturnTrace()) |trace| {
                        std.debug.dumpErrorReturnTrace(trace);
                    }
                    if (fail_first) {
                        break;
                    }
                },
            }
        }

        printer.status(status, "[{s}]\n", .{@tagName(status)});
    }

    const total_tests = pass + fail;
    const status: Status = if (fail == 0) .pass else .fail;
    printer.status(status, "\n{d} of {d} test{s} passed\n", .{ pass, total_tests, if (total_tests != 1) "s" else "" });
    if (skip > 0) {
        printer.status(.skip, "{d} test{s} skipped\n", .{ skip, if (skip != 1) "s" else "" });
    }
    if (leak > 0) {
        printer.status(.fail, "{d} test{s} leaked\n", .{ leak, if (leak != 1) "s" else "" });
    }
    std.process.exit(if (fail == 0) 0 else 1);
}

const Printer = struct {
    fn init() Printer {
        return .{};
    }

    fn fmt(self: Printer, comptime format: []const u8, args: anytype) void {
        _ = self; // autofix
        std.debug.print(format, args);
    }

    fn status(self: Printer, s: Status, comptime format: []const u8, args: anytype) void {
        const color = switch (s) {
            .pass => "\x1b[32m",
            .fail => "\x1b[31m",
            .skip => "\x1b[33m",
            else => "",
        };
        std.debug.print("{s}", .{color});
        std.debug.print(format, args);
        self.fmt("\x1b[0m", .{});
    }
};

std.os.exit is not portable. A simple fix is using std.process.exit

Replace

std.os.exit(if (fail == 0) 0 else 1);

To

std.process.exit(if (fail == 0) 0 else 1);

And another fix:

const test_name = t.name[5..];

to

const test_name = t.name;

After adding the comments that @fanyang89 suggests, the runner works great. This is pretty helpful on smaller projects. Great work @nurpax

One thing that also needs to change is in zig 0.13.0 you need to change test_runner.zig to b.path("test_runner.zig")

Here is an example

    const exe_unit_tests = b.addTest(.{
        .root_source_file = b.path("src/test.zig"),
        .target = target,
        .optimize = optimize,
        .test_runner = b.path("src/test_runner.zig"),
    });

Thanks everyone for the suggestions! I updated the original source with the fixes from comments.

What's the license on this?

Good question. It's adapted from https://gist.github.com/karlseguin/c6bea5b35e4e8d26af6f81c22cb5d76b. I'd be ok with MIT.

I wish something like this was included in Zig's own test runner.

I asked there too. Without a license I can't really use it for anything real unfortunately, but it's nice to know that this one will become MIT if the other one picks a compatible license.

Upstream project was clarified as being MIT \o/

hello i try to use this in zig 0.14.0using above snippets but i get an error

 error: expected type '?Build.Step.Compile.TestRunner', found 'Build.LazyPath'
        .test_runner = b.path("test_runner.zig"),
        ~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~

so right now i pass the custom runner like this and it works fine

    const project_unit_tests = b.addTest(.{
        .root_source_file = b.path(b.pathJoin(&.{"src", "test.zig"})),
        .target = target,
        .optimize = optimize
    });
    project_unit_tests.test_runner = .{
        .path = b.path("test_runner.zig"),
        .mode = .simple
    };

i am still new to zig so if folks right here have better solution, feels free to correct my workaround

Thanks @mamahnxarya201, your change is fine. I updated the gist content. It should work with zig-0.14 now.

This no longer works for a recent zig master version (mine: 0.15.0-dev.1034+bd97b6618). My adapted, ported and working version is here: https://github.com/utensil/native-land/blob/main/yard-zig/basic-xp/test_runner.zig

It also added some features and tweaks, before the port. It looks like this for zig build test --summary all:

// omitted....
simd.test.vector swizzle operations [pass]
simd.test.vector min/max operations [pass]
simd.test.vector bitwise operations [pass]
simd.test.vector floating point operations [pass]
simd.test.vector load and store operations [pass]
simd: 12 of 12 tests passed
string_literals.test.multiline string literals vs regular strings [pass]
string_literals.test.multiline strings preserve backslashes [pass]
string_literals.test.multiline strings for code generation [pass]
string_literals.test.multiline strings preserve trailing whitespace [pass]
string_literals.test.@embedFile vs multiline strings [pass]
string_literals: 5 of 5 tests passed
Total: 83 of 83 tests passed
Build Summary: 5/5 steps succeeded
test success
+- run test success 243ms MaxRSS:2M
|  +- compile test Debug native success 842ms MaxRSS:282M
+- run test success 239ms MaxRSS:1M
   +- compile test Debug native success 719ms MaxRSS:244M

Disclaimer: Some code is coded with the help of a coding agent. It figured out how to fix the errors by trying a few different ways. I checked the latest document and nudged it in the right direction.

Thanks @utensil! I updated to gist to use std.debug.print() (like in your version) and it should now work on zig-0.14 and 0.15.0-dev.1092+d772c0627. Using std.debug.print is probably a safer path to keep this compiling than updating to use the new Io interface. (Honestly, I couldn't get the old std.fs.File based code to compile. ;))

Updated to zig 0.16.0.