file-handler.diff

diff --git a/python/cudf_polars/cudf_polars/experimental/benchmarks/utils.py b/python/cudf_polars/cudf_polars/experimental/benchmarks/utils.py
index f40889b68c..375e1dbebb 100644
--- a/python/cudf_polars/cudf_polars/experimental/benchmarks/utils.py
+++ b/python/cudf_polars/cudf_polars/experimental/benchmarks/utils.py
@@ -1569,9 +1569,9 @@ def run_polars_query(
 
     for i in range(args.iterations):
         if _HAS_STRUCTLOG and run_config.collect_traces:
-            setup_logging(q_id, i)
+            setup_logging(q_id, i, run_config.run_id)
             if client is not None:
-                client.run(setup_logging, q_id, i)
+                client.run(setup_logging, q_id, i, run_config.run_id)
 
         try:
             record = run_polars_query_iteration(
@@ -1995,7 +1995,7 @@ def run_polars_ray(
     sys.exit(1 if (query_failures or validation_failures) else 0)
 
 
-def setup_logging(query_id: int, iteration: int) -> None:  # noqa: D103
+def setup_logging(query_id: int, iteration: int, run_id: uuid.UUID) -> None:  # noqa: D103
     import cudf_polars.dsl.tracing
 
     if not cudf_polars.dsl.tracing.LOG_TRACES:
@@ -2049,15 +2049,22 @@ def setup_logging(query_id: int, iteration: int) -> None:  # noqa: D103
         json_renderer = structlog.processors.JSONRenderer()
 
         stream = io.StringIO()
-        json_file_handler = logging.StreamHandler(stream)
-        json_file_handler.setFormatter(
-            structlog.stdlib.ProcessorFormatter(
-                processor=json_renderer,
-                foreign_pre_chain=shared_processors,
-            )
+        memory_handler = logging.StreamHandler(stream)
+        os.makedirs(f"logs/{run_id}", exist_ok=True)
+        ndjson_handler = logging.FileHandler(
+            f"logs/{run_id}/{os.getpid()}.ndjson", mode="a", encoding="utf-8"
+        )
+        proc_fmt = structlog.stdlib.ProcessorFormatter
+        memory_handler.setFormatter(
+            proc_fmt(processor=json_renderer, foreign_pre_chain=shared_processors)
+        )
+        ndjson_handler.setFormatter(
+            proc_fmt(processor=json_renderer, foreign_pre_chain=shared_processors)
         )
 
-        logging.basicConfig(level=logging.INFO, handlers=[json_file_handler])
+        logging.basicConfig(
+            level=logging.INFO, handlers=[memory_handler, ndjson_handler]
+        )
 
         structlog.configure(
             processors=[