miniriot.ml

miniriot.ml

let sample_fun () =
  Format.printf "creating fragmented data\r\n%!";
  let make_data () =
    let smol_array = Array.make 1000 "" in
    let big_array = Array.make 1000 "" in
    for i = 0 to 999 do
      if i mod 2 = 0 then smol_array.(i) <- String.make 1 'h'
      else big_array.(i) <- String.make 500_000 'x'
    done;
    smol_array
  in
  let smol_array = make_data () in
  Format.printf "size: %d bytes\r\n%!" (Array.length smol_array);
  ()

let spawn, next_fiber =
  let lock = Mutex.create () in
  let q = Queue.create () in
  let push_fiber fn = Mutex.protect lock @@ fun () -> Queue.push fn q in
  let next_fiber () = Mutex.protect lock @@ fun () -> Queue.take_opt q in
  (push_fiber, next_fiber)

let rec run_scheduler () =
  Unix.sleepf 0.01;
  match next_fiber () with
  | None -> 
      (* Gc.full_major (); *)
      run_scheduler ()
  | Some fn ->
      let () = fn () in
      run_scheduler ()

let _run_on_one_thread () =
  for _i = 0 to 100 do
    spawn sample_fun
  done;
  run_scheduler ()

let multicore () =
  let nproc = Domain.recommended_domain_count () - 1 in
  let domains = List.init nproc (fun _id -> Domain.spawn run_scheduler) in
  for _i = 0 to 100 do
    spawn sample_fun
  done;
  List.iter Domain.join domains

let () = multicore ()

Quick overview/analysis:

• sample_fun makes two arrays, big_array with 500 newly-allocated strings of length 500K bytes each, and smol_array with 500 newly-allocated strings of length 1 byte each. Each array also has the array block itself, which is 1000 words (8K on a 64-bit platform). sample_fun then exits, discarding first big_array and then (after a very small amount of additional allocation) smol_array. However, it has allocated so much data (250MB, near enough), almost all of that data has been promoted into the major heap.
• There's a simple mutex-protected queue, used to run this function 100 times, across a number of domains functioning as a worker pool.
• these domains never terminate; when they run out of work to do they just loop tail-recursively in run_scheduler, without allocating.

You say on Slack that "i'm running this program on a rather beefy machine (64-cores, 64gigs of ram)", so I'm going to assume that recommended_domain_count() is 64-ish. So (plausibly) about half the domains run sample_fun once, and the other half run it twice, more-or-less synchronously (handwave here).

So, plausibly (a) a major GC is not getting scheduled (or possibly not running to completion) after that second round of domains do their allocation work, so (b) each of those domains ends up with 250M-ish in its major heap, for a total heap size of roughly 8 GiB, and (c) after that, all the domains are idle, and none of them are allocating, so no GC gets scheduled. The main domain is just waiting in Domain.join on the first worker domain.

After a domain discards big_array, it doesn't do any more allocation (or only a trivial amount), so the usual allocation-driven GC scheduling has little to go on.