some tcp test runner I toyed with with the experimental network implementation

tcp.diff

diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig
index d71f2b9274..c2afa500aa 100644
--- a/lib/std/special/test_runner.zig
+++ b/lib/std/special/test_runner.zig
@@ -1,35 +1,353 @@
+//! Default test runner
+//! assume: OS environment
+//! assume: (sub)processes and multithreading possible
+//! assume: IPC via pipes possible
+//! assume: The order of test execution **must not matter**.
+//!         User ensures global state is properly initialized and
+//!         reset/cleaned up between test blocks.
+//! assume: User does not need additional messages besides test_name + number
+//! assume: ports 9084, 9085 usable for localhost tcp (Zig Test, ascii ZT: 90, 84)
+//! assert: User writes only 1 expectPanic per test block.
+//!         Improving this requires to keep track of expect* functions in test blocks
+//!         in Compilation.zig and have a convention how to initialize shared state.
+
+// unclear: are 2 binaries needed or can 1 binary have 2 panic handlers?
+//          could the panic handler be given information where to write the panic?
+//          should we generalize this concept at comptime?
+// unclear: dedicated API or other checks to ensure things gets cleaned up?
+
+// principle:
+// 2 binaries: compiler and test_runner
+// 1. compiler compiles main() in this file as test_runner
+// 2. compiler spawns test_runner as subprocess
+// 3. test_runner (control) spawns itself as subprocess (worker)
+// 4. on panic or finishing, message is written to socket for control to read
+//    (worker has a custom panic handler to write panic message to the socket)
+// communication via 2 sockets:
+//     - 1. testnumber testnumber_exitstatus msglen
+//     - 2. testout with data
+// copying the memory from tcp socket to pipe for error message is still necessary
+// https://stackoverflow.com/questions/11847793/are-tcp-socket-handles-inheritable
+
+// compiler
+//    |  --spawns-->  control
+//    |                  |       --spawn at testfn0-->                worker
+//    |                  |      <--testout: testfn0_exit_status 0--      |
+//    |                  |      <--testout: testfn1_exit_status 0--      |
+//    |                  |                 ...                           |
+//    |                  |      <--testout: testfnx_exit_status msglen-- |
+//    |                  |      <--stdout: panic_msg--                   |
+//    | (<--panic_msg--) |                                               |
+//    |                  |      (--spawn at testfnx+1-->)                |
+//    |                  |                                               |
+// unclear: should we special case 1.panic_msg, 2. panic_msg+context ?
+
 const std = @import("std");
 const io = std.io;
 const builtin = @import("builtin");
 
+const net = std.x.net;
+const os = std.x.os;
+const ip = net.ip;
+const tcp = net.tcp;
+const IPv4 = os.IPv4;
+const IPv6 = os.IPv6;
+const Socket = os.Socket;
+const Buffer = os.Buffer;
+const have_ifnamesize = @hasDecl(std.os.system, "IFNAMESIZE"); // interface namesize
+const ChildProcess = std.ChildProcess;
+
 pub const io_mode: io.Mode = builtin.test_io_mode;
 
 var log_err_count: usize = 0;
 
-var args_buffer: [std.fs.MAX_PATH_BYTES + std.mem.page_size]u8 = undefined;
+// TODO REVIEW: why is not std.math.max(3*MAX_PATH_BYTES, std.mem.page_size) used ?
+// paths arg[0], arg[1], --worker --start-index cwd u64_max_string
+// with each using as worst case MAX_PATH_BYTES
+// TODO reserved size into libstd
+// TODO tcp code sizes (provide stubs for worst case string representation)
+
+// zig fmt: off
+const reserved_size = std.math.max(CtrlMsg.maxsize_testnr
+    + CtrlMsg.maxsize_exit + CtrlMsg.maxsize_len
+    + 3 * std.fs.MAX_PATH_BYTES, std.mem.page_size);
+// zig fmt: on
+
+var args_buffer: [reserved_size]u8 = undefined;
 var args_allocator = std.heap.FixedBufferAllocator.init(&args_buffer);
 
-fn processArgs() void {
+const State = enum {
+    Control,
+    Worker,
+};
+
+const Cli = struct {
+    state: State,
+    start_index: u64,
+};
+
+// running on raw sockets is not possible without additional permissions
+// https://squidarth.com/networking/systems/rc/2018/05/28/using-raw-sockets.html
+// only alternative: tcp/udp
+const TcpIo = struct {
+    state: State,
+    // ip always localhost (ie 127.0.0.1)
+    port_ctrl: u16 = 9084,
+    port_data: u16 = 9085,
+    ctrl: union {
+        listener: tcp.Listener,
+        client: tcp.Client,
+    },
+    data: union {
+        listener: tcp.Listener,
+        client: tcp.Client,
+    },
+    addr_ctrl: ip.Address,
+    addr_data: ip.Address,
+    conn_ctrl: tcp.Connection,
+    conn_data: tcp.Connection,
+
+    fn setup(state: State) !TcpIo {
+        var tcpio = TcpIo{
+            .state = state,
+            .ctrl = undefined,
+            .data = undefined,
+            .addr_ctrl = undefined,
+            .addr_data = undefined,
+            .conn_ctrl = undefined,
+            .conn_data = undefined,
+        };
+
+        //set SO_REUSEADDR for the port to prevent .ADDRINUSE errors on fast invocations
+        //problem: Multiple instances could exist and listen to another messages
+        //solution: have a test_runner server running on localhost
+        switch (state) {
+            State.Control => {
+                tcpio.ctrl = .{ .listener = try tcp.Listener.init(.ip, .{ .close_on_exec = true }) };
+                tcpio.data = .{ .listener = try tcp.Listener.init(.ip, .{ .close_on_exec = true }) };
+                try tcpio.ctrl.listener.setReuseAddress(true);
+                try tcpio.data.listener.setReuseAddress(true);
+                try tcpio.ctrl.listener.bind(ip.Address.initIPv4(IPv4.unspecified, tcpio.port_ctrl));
+                try tcpio.data.listener.bind(ip.Address.initIPv4(IPv4.unspecified, tcpio.port_data));
+                try tcpio.ctrl.listener.listen(1);
+                try tcpio.data.listener.listen(1);
+                tcpio.addr_ctrl = try tcpio.ctrl.listener.getLocalAddress();
+                tcpio.addr_data = try tcpio.data.listener.getLocalAddress();
+                switch (tcpio.addr_ctrl) {
+                    .ipv4 => |*ipv4| ipv4.host = IPv4.localhost,
+                    .ipv6 => unreachable,
+                }
+                switch (tcpio.addr_data) {
+                    .ipv4 => |*ipv4| ipv4.host = IPv4.localhost,
+                    .ipv6 => unreachable,
+                }
+            },
+            State.Worker => {
+                tcpio.ctrl = .{ .client = try tcp.Client.init(.ip, .{ .close_on_exec = true }) };
+                tcpio.data = .{ .client = try tcp.Client.init(.ip, .{ .close_on_exec = true }) };
+                const s_localhost = "127.0.0.1"; // HACK around libstd
+                const localhost = try IPv4.parse(s_localhost);
+                tcpio.addr_ctrl = ip.Address.initIPv4(localhost, tcpio.port_ctrl);
+                tcpio.addr_data = ip.Address.initIPv4(localhost, tcpio.port_data);
+            },
+        }
+        return tcpio;
+    }
+};
+
+// args 0:testbinary, 1:compilerbinary, [2: --worker]
+fn processArgs() Cli {
     const args = std.process.argsAlloc(args_allocator.allocator()) catch {
         @panic("Too many bytes passed over the CLI to the test runner");
     };
-    if (args.len != 2) {
-        const self_name = if (args.len >= 1) args[0] else if (builtin.os.tag == .windows) "test.exe" else "test";
-        const zig_ext = if (builtin.os.tag == .windows) ".exe" else "";
+    const self_name = if (args.len >= 1) args[0] else if (builtin.os.tag == .windows) "test.exe" else "test";
+    const zig_ext = if (builtin.os.tag == .windows) ".exe" else "";
+    if (args.len < 2 or 5 < args.len) {
         std.debug.print("Usage: {s} path/to/zig{s}\n", .{ self_name, zig_ext });
         @panic("Wrong number of command line arguments");
     }
+    var cli = Cli{
+        .state = State.Control,
+        .start_index = 0,
+    };
+    if (args.len == 3 or args.len == 5) {
+        if (!std.mem.eql(u8, args[2], "--worker")) {
+            std.debug.print("Usage: {s} path/to/zig{s}\n", .{ self_name, zig_ext });
+            @panic("Found args[2] != '--worker'");
+        }
+        cli.state = State.Worker;
+    }
+    if (args.len == 4) {
+        if (!std.mem.eql(u8, args[2], "--start-index")) {
+            std.debug.print("Usage: {s} path/to/zig{s}\n", .{ self_name, zig_ext });
+            @panic("Found args[2] != '--start-index'");
+        }
+        cli.start_index = std.fmt.parseUnsigned(u64, args[3], 10) catch unreachable;
+    }
+    if (args.len == 5) {
+        if (!std.mem.eql(u8, args[3], "--start-index")) {
+            std.debug.print("Usage: {s} path/to/zig{s}\n", .{ self_name, zig_ext });
+            @panic("Found args[2] != '--start-index'");
+        }
+        cli.start_index = std.fmt.parseUnsigned(u64, args[4], 10) catch unreachable;
+    }
+    std.testing.test_runner_exe_path = args[0];
     std.testing.zig_exe_path = args[1];
+    return cli;
 }
 
-pub fn main() void {
+fn maxAsciiDigits(comptime UT: type) u64 {
+    return std.fmt.count("{d}", .{std.math.maxInt(UT)});
+}
+
+// count(comptime fmt: []const u8, args: anytype) {c} ascii string
+//pub fn count(comptime fmt: []const u8, args: anytype) u64 {
+const CtrlMsg = struct {
+    const Self = @This();
+    testnr: u32,
+    exit: u8, // exit_status
+    len: u32,
+    const maxsize_testnr: u64 = maxAsciiDigits(std.meta.fieldInfo(CtrlMsg, .testnr).field_type);
+    const maxsize_exit: u64 = maxAsciiDigits(Self.exit);
+    const maxsize_len: u64 = maxAsciiDigits(Self.len);
+    // writes into padded msgbuf
+    //fn print(self: *CtrlMsg, msgbuf: []u8) !void {
+    //    // TODO write into buffer
+    //    _ = ctrlmsg;
+    //    _ = msgbuf;
+    //}
+    // reads from padded msgbuf
+    //fn parse(self: *CtrlMsg, msgbuf: []u8) !void {
+    //    _ = ctrlmsg;
+    //    _ = msgbuf;
+    //}
+};
+
+// args: path_to_testbinary, path_to_zigbinary, [--worker, --start-index 0]
+pub fn main() !void {
+    if (!have_ifnamesize) return error.FAILURE;
+
     if (builtin.zig_backend != .stage1 and
         (builtin.zig_backend != .stage2_llvm or builtin.cpu.arch == .wasm32))
     {
         return main2() catch @panic("test failure");
     }
-    if (builtin.zig_backend == .stage1) processArgs();
+    var cli = processArgs();
+    var tcpio = TcpIo.setup(cli.state) catch unreachable;
+
     const test_fn_list = builtin.test_functions;
+
+    const cwd = std.process.getCwdAlloc(args_allocator.allocator()) catch {
+        @panic("Too many bytes passed over the CLI to the test runner");
+    }; // windows compatibility requires allocation
+    std.debug.print("test_runner_exe_path zig_exe_path cli.start_index\n", .{});
+    std.debug.print("{s} {s} --start-index {d}\n", .{ std.testing.test_runner_exe_path, std.testing.zig_exe_path, cli.start_index });
+    std.debug.print("--cwd: {s}\n", .{cwd});
+
+    // len("2**64-1")= len("18446744073709551615") = 20
+    // TODO add constants to libstd
+    var buf_start_index = std.mem.Allocator.alloc(args_allocator.allocator(), u8, 20) catch unreachable;
+
+    switch (tcpio.state) {
+        State.Control => {
+            while (cli.start_index < test_fn_list.len) {
+                const s_start_index = std.fmt.bufPrint(buf_start_index[0..], "{d}", .{cli.start_index}) catch unreachable;
+                const args = [_][]const u8{
+                    std.testing.test_runner_exe_path,
+                    std.testing.zig_exe_path,
+                    "--worker",
+                    "--start-index",
+                    s_start_index,
+                };
+                var child_proc = try ChildProcess.init(&args, std.testing.allocator);
+                defer child_proc.deinit();
+                try child_proc.spawn();
+                std.debug.print("child_proc spawned:\n", .{});
+                std.debug.print("args {s}\n", .{args});
+
+                tcpio.conn_ctrl = try tcpio.ctrl.listener.accept(.{ .close_on_exec = true }); // blocking
+                tcpio.conn_data = try tcpio.data.listener.accept(.{ .close_on_exec = true }); // blocking
+                defer tcpio.conn_ctrl.deinit();
+                defer tcpio.conn_data.deinit();
+                std.debug.print("connections succesful\n", .{});
+
+                //CtrlMsg
+
+                //const msg = parseMessage(ctrl, );
+                // TODO reading + parsing messages from socket
+
+                //const message = "hello world";
+                //var buf: [message.len + 1]u8 = undefined;
+                //var msg = Socket.Message.fromBuffers(&[_]Buffer{
+                //    Buffer.from(buf[0 .. message.len / 2]),
+                //    Buffer.from(buf[message.len / 2 ..]),
+                //});
+                //_ = try tcpio.conn_ctrl.client.readMessage(&msg, 0);
+                //try std.testing.expectEqualStrings(message, buf[0..message.len]);
+                //std.debug.print("comparison successful\n", .{});
+
+                //const ret_val = child_proc.wait();
+                //try std.testing.expectEqual(ret_val, .{ .Exited = 0 });
+                //std.debug.print("server exited\n", .{});
+
+                // respawn child_process until we reach test_fn.len:
+                //   after wait():
+                //   if ret_val == 0
+                //     print OK of all messages
+                //   else
+                //     if messages in ctrl empty:
+                //       print unexpected panic during test: got no panic message(s)
+                //     else
+                //       if message == expected_message
+                //         print OK of current test_fn (other OKs were printed by child_process)
+                //         continue;
+                //       else
+                //         print fatal, got 'message', expected 'expected_message'
+                //
+                //
+                // panic message formatting:
+                // 1. panic message must be allocated
+                // 2. ctrl: test_fn_number exit_code msglen msg ?
+                // 3. alternative is to use data: msg
+
+                //parse executed test into cur_index
+                //update cli.start_index by checking, if start_index + 1 == cur_index
+                cli.start_index += 1; // update
+            }
+        },
+        State.Worker => {
+            try tcpio.ctrl.client.connect(tcpio.addr_ctrl);
+            try tcpio.data.client.connect(tcpio.addr_data);
+
+            const message = "hello world";
+            _ = try tcpio.ctrl.client.writeMessage(Socket.Message.fromBuffers(&[_]Buffer{
+                Buffer.from(message[0 .. message.len / 2]),
+                Buffer.from(message[message.len / 2 ..]),
+            }), 0);
+
+            var index = cli.start_index;
+            while (index < test_fn_list.len) : (index += 1) {
+                std.debug.print("{d} {s}\n", .{ index, test_fn_list[index].name });
+
+                const result = test_fn_list[index].func();
+                if (result) |_| {
+                    std.debug.print("OK\n", .{});
+                    // TODO write to socket
+                } else |err| switch (err) {
+                    error.SkipZigTest => {
+                        std.debug.print("SKIP\n", .{});
+                    },
+                    else => {
+                        std.debug.print("FAIL\n", .{});
+                    },
+                }
+            }
+        },
+    }
+
+    for (test_fn_list) |test_fn, i|
+        std.debug.print("{d} {s}\n", .{ i, test_fn.name });
+
     var ok_count: usize = 0;
     var skip_count: usize = 0;
     var fail_count: usize = 0;
@@ -77,6 +395,7 @@ pub fn main() void {
                 continue;
             },
         } else test_fn.func();
+
         if (result) |_| {
             ok_count += 1;
             test_node.end();
diff --git a/lib/std/special/todos b/lib/std/special/todos
new file mode 100644
index 0000000000..ed77744753
--- /dev/null
+++ b/lib/std/special/todos
@@ -0,0 +1,81 @@
+next: code integration
+
+1. list of functions + expected status code (panic or no panic)
+2. make status codes configurable?
+3. test.expectPanic in libstd (expectPanic failure => return error.NoPanic)
+4. noreturn after test.expectPanic code branch
+5. figuring out the corner cases of various kernels etc
+
+test_runner.zig:
+
+- [x] document default test_runner assumptions
+- [x] document default test_runner interaction that is planned
+- [x] prototype tcp communication between server and client
+- [ ] args parsing
+- [ ] writing results from tcp
+- [ ] expectPanic with adding noreturn, if possible
+- [ ] panic message encoding in worker
+- [ ] panic message checking in control
+- [ ] panic comparison to expected result
+
+not in scope of PR:
+- [ ] plan for parallelized execution of test blocks or how this should work in build.zig?
+      must be user-written and make test_runner unnecessary complex
+      simpler: test_runner libraries for user-crafting of test runner. :)
+      does the same go for panic handler?
+- [ ] "portable inspired signal communication via pipes" (strip down POSIX signals)?
+- [ ] complicates test_runner => should this be another test_runner?
+- [ ] how can we come up with a test runner lib to customize your own?
+      * what stuff is strictly necessary?
+      * what stuff is not necessary?
+      * how to minimize maintenance cost, while ensuring all necessary necessary edge cases are tested?
+
+- have a way to list+enumerate all tests inclusive their respective test blocks
+  * done:
+    for (test_fn_list) |test_fn, i|
+        std.debug.print("{d} {s}\n", .{ i, test_fn.name });
+
+- figure out how to create another pipe between processes for the control plane
+(stdin,stdout and stderr are for data only! no complex parsing shennanigans or signaling etc
+unless there is significant performance gain)
+--verbose or --debug to let each process write with timestamp to file
+=> figure out what the Kernel provides (multithreading works, but multiprocesses?)
+TODO spawn subprocess to write status
+
+- custom panic handler + cleanup routine (try not to deal with stage1 having weird behavior
+User-defined test runner #6621 (comment))
+
+
+----- PERSONAL NOTES -----
+It is now possible to 1. build tests without running them and 2. using custom test commands, 3. switching the test runner which is sufficient to get things working:
+```txt
+    \\Test Options:
+    \\  --test-filter [text]           Skip tests that do not match filter
+    \\  --test-name-prefix [text]      Add prefix to all tests
+    \\  --test-cmd [arg]               Specify test execution command one arg at a time
+    \\  --test-cmd-bin                 Appends test binary path to test cmd args
+    \\  --test-evented-io              Runs the test in evented I/O mode
+    \\  --test-no-exec                 Compiles test binary without running it
+```
+
+What is missing potential are nice to haves, but complicate things
+1. benchmarks (e.g. if the function name starts with "bench_")
+2. parallel execution
+3. testing time
+1. Meaningful benchmarks require system control (either idle) and have no complexity limit,
+so they are a bad target for functional tests inside continuous integration and are
+better used in build.zig after establishing necessary conditions.
+2. Parallel execution probably means multi-threading,
+which can influence memory stuff in the same process. It is only useful,
+if potential memory problems are explicitly accepted by the user.
+3. Testing time is captured by the CI or can easily be captured by `build.zig`,
+so it has no additional benefit.
+To me point 2 sounds worth maintaining inside libstd for imminent user-interaction
+inside the REPL, but no complexity+maintenance estimation vs the benefit was given,
+so I suspect limited applicability to offer parallelization of test execution
+inside a process.
+To me the solution for custom runners looks like a hack to set a file as `root`
+from within the file, which makes the behavior from `build.zig` non-trivial since
+it doesnt execute what the reader of `build.zig` expects.
+If the user wants a custom test runner, the user can use aforementioned commands
+to build and run the test binary.
diff --git a/lib/std/testing.zig b/lib/std/testing.zig
index 174e898bca..cf969b9b6c 100644
--- a/lib/std/testing.zig
+++ b/lib/std/testing.zig
@@ -18,8 +18,12 @@ pub var base_allocator_instance = std.heap.FixedBufferAllocator.init("");
 /// TODO https://github.com/ziglang/zig/issues/5738
 pub var log_level = std.log.Level.warn;
 
-/// This is available to any test that wants to execute Zig in a child process.
-/// It will be the same executable that is running `zig test`.
+/// This is only available inside test blocks.
+/// See ./special/test_runner.zig for more details on the default test runner.
+pub var test_runner_exe_path: []const u8 = undefined;
+
+/// This is only available inside test blocks.
+/// See ./special/test_runner.zig for more details on the default test runner.
 pub var zig_exe_path: []const u8 = undefined;
 
 /// This function is intended to be used only in tests. It prints diagnostics to stderr
@@ -571,7 +575,7 @@ fn printLine(line: []const u8) void {
     print("{s}\n", .{line});
 }
 
-test {
+test "expectEqualStrings" {
     try expectEqualStrings("foo", "foo");
 }
 
diff --git a/test.zig b/test.zig
new file mode 100644
index 0000000000..72b4d9d673
--- /dev/null
+++ b/test.zig
@@ -0,0 +1,7 @@
+// call via
+// ./o/0fdce2e04dcd32f91f3ca6796d625f4a/test ../build/zig
+
+test "name1" {}
+test "name2" {}
+test "name3" {}
+test "name4" {}