Small benchmark to deduplicate /nix/store into a bup repository

nix-store-bup-benchmark.py

#! /usr/bin/env python3

import glob
import os
import subprocess
import sys
from itertools import zip_longest

# From https://stackoverflow.com/questions/434287/what-is-the-most-pythonic-way-to-iterate-over-a-list-in-chunks/434411#434411
def grouper(iterable, n):
    args = [iter(iterable)] * n
    # remove trailing `None`s
    return ((x for x in it if x is not None) for it in zip_longest(*args))

store_paths = glob.glob('/nix/store/*')

# Pass this many paths to 1 `tar` invocation to reduce `bup` invocations.
num_batch_paths = 100

num_processed = 0
for group in grouper(store_paths, num_batch_paths):
  print(num_processed)
  paths = list(group)
  for p in paths:
    print(p)
  command = "tar c {} | bup split -n nix-store".format(' '.join(paths))
  env = os.environ
  env["BUP_DIR"] = "nix-store-bup-dedup-test-tar"
  subprocess.run(command, shell=True, env=env)
  print()
  num_processed += num_batch_paths