benchmarking the zig tokenizer

output.txt

[nix-shell:~/Downloads/zig/build-release]$ ./zig build-exe test.zig -OReleaseFast

[nix-shell:~/Downloads/zig/build-release]$ ls -hl ../lib/std/special/compiler_rt/udivmod{t,d}i4_test.zig test.zig
-rw-r--r-- 1 andy users 1.9M May 10 21:29 ../lib/std/special/compiler_rt/udivmoddi4_test.zig
-rw-r--r-- 1 andy users 9.8M May 10 21:29 ../lib/std/special/compiler_rt/udivmodti4_test.zig
-rw-r--r-- 1 andy users 1.2K Jul  3 16:46 test.zig

[nix-shell:~/Downloads/zig/build-release]$ ./test test.zig
tokenizer:         25 MiB/s

[nix-shell:~/Downloads/zig/build-release]$ ./test ../lib/std/special/compiler_rt/udivmoddi4_test.zig 
tokenizer:        312 MiB/s

[nix-shell:~/Downloads/zig/build-release]$ ./test ../lib/std/special/compiler_rt/udivmodti4_test.zig 
tokenizer:        585 MiB/s

tokenizer_bench.zig

const std = @import("std");
const builtin = std.builtin;
const mem = std.mem;
const time = std.time;
const Timer = time.Timer;

const KiB = 1024;
const MiB = 1024 * KiB;

pub fn main() !void {
    const stdout = std.io.getStdOut().writer();

    var arena_allocator = std.heap.ArenaAllocator.init(std.heap.page_allocator);
    const arena = &arena_allocator.allocator;
    const args = try std.process.argsAlloc(arena);

    const source_file = args[1];
    const source = try std.fs.cwd().readFileAllocOptions(
        arena,
        source_file,
        std.math.maxInt(usize),
        null,
        1,
        0,
    );

    var timer = try Timer.start();
    const start = timer.lap();

    var tokenizer = std.zig.Tokenizer.init(source);
    var hash: usize = 0;
    while (true) {
        const token = tokenizer.next();
        hash +%= @enumToInt(token.tag);
        if (token.tag == .eof) break;
    }
    mem.doNotOptimizeAway(&hash);
    const end = timer.read();

    const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s;
    const throughput = @floatToInt(u64, @intToFloat(f64, source.len) / elapsed_s);

    try stdout.print("tokenizer: {:10} MiB/s\n", .{throughput / (1 * MiB)});
}