Sandlock vs Docker Redis benchmark

bench_redis.py

#!/usr/bin/env python3
"""Benchmark Redis: bare metal vs sandlock vs Docker.

redis-server runs in each environment.
redis-benchmark runs on host, connecting to the server.
Tests SET/GET with small payloads (256 bytes) — similar to RioDB's pattern.
"""

import json
import subprocess
import sys
import time

PORT = 16379
REQUESTS = 100000
CLIENTS = 50
DATA_SIZE = 256

def run(cmd, **kwargs):
    return subprocess.run(cmd, capture_output=True, text=True, **kwargs)

def kill_server():
    run(["pkill", "-f", f"redis-server.*:{PORT}"])
    run(["docker", "rm", "-f", "redis-bench"])
    time.sleep(0.5)

def run_benchmark(host="127.0.0.1"):
    """Run redis-benchmark, return parsed results."""
    result = run([
        "redis-benchmark",
        "-h", host, "-p", str(PORT),
        "-n", str(REQUESTS),
        "-c", str(CLIENTS),
        "-d", str(DATA_SIZE),
        "-t", "set,get",
        "--csv",
    ], timeout=120)
    # CSV: "test","rps","avg_latency_ms","min_latency_ms","p50_latency_ms",
    #       "p95_latency_ms","p99_latency_ms","max_latency_ms"
    lines = result.stdout.strip().split("\n")
    results = {}
    for line in lines:
        parts = line.replace('"', '').split(",")
        if parts[0] in ("SET", "GET"):
            results[parts[0]] = {
                "rps": float(parts[1]),
                "avg_ms": float(parts[2]),
                "p50_ms": float(parts[4]),
                "p99_ms": float(parts[6]),
            }
    return results


def bench_bare():
    """Redis server on host, no isolation."""
    kill_server()
    server = subprocess.Popen(
        ["redis-server", "--port", str(PORT), "--save", "",
         "--appendonly", "no", "--loglevel", "warning"],
        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
    )
    time.sleep(0.5)
    results = run_benchmark()
    server.terminate()
    server.wait(timeout=5)
    return results


def bench_sandlock():
    """Redis server inside sandlock."""
    kill_server()
    server = subprocess.Popen(
        [sys.executable, "-c", f"""
from sandlock import Sandbox, Policy
policy = Policy(
    fs_readable=["/usr", "/lib", "/lib64", "/etc", "/bin", "/sbin", "/proc", "/dev", "/tmp"],
    fs_writable=["/tmp"],
    net_bind=[{PORT}],
    net_connect=[{PORT}],
)
Sandbox(policy).run([
    "redis-server", "--port", "{PORT}", "--save", "",
    "--appendonly", "no", "--loglevel", "warning",
])
"""],
        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
    )
    time.sleep(2)
    results = run_benchmark()
    server.terminate()
    try:
        server.wait(timeout=5)
    except subprocess.TimeoutExpired:
        server.kill()
    return results


def bench_docker():
    """Redis server inside Docker container, same binary as host."""
    kill_server()
    run(["docker", "run", "-d", "--rm",
         "--name", "redis-bench",
         "-p", f"{PORT}:{PORT}",
         "-v", "/usr/bin/redis-server:/usr/bin/redis-server:ro",
         "-v", "/usr/lib:/usr/lib:ro",
         "-v", "/lib:/lib:ro",
         "ubuntu:22.04",
         "redis-server", "--port", str(PORT), "--save", "",
         "--appendonly", "no", "--protected-mode", "no", "--loglevel", "warning"])
    time.sleep(3)
    results = run_benchmark()
    run(["docker", "stop", "-t", "2", "redis-bench"])
    return results


def print_header():
    print(f"  {'':12s} {'SET rps':>10s}  {'GET rps':>10s}  "
          f"{'SET p50':>8s}  {'SET p99':>8s}  {'GET p50':>8s}  {'GET p99':>8s}")
    print(f"  {'-'*12} {'-'*10}  {'-'*10}  "
          f"{'-'*8}  {'-'*8}  {'-'*8}  {'-'*8}")

def print_row(name, r, bare=None):
    s, g = r["SET"], r["GET"]
    pct = ""
    if bare:
        pct = f"  ({(s['rps']+g['rps'])/(bare['SET']['rps']+bare['GET']['rps'])*100:.1f}%)"
    print(f"  {name:<12s} {s['rps']:>10,.0f}  {g['rps']:>10,.0f}  "
          f"{s['p50_ms']:>7.3f}ms {s['p99_ms']:>7.3f}ms "
          f"{g['p50_ms']:>7.3f}ms {g['p99_ms']:>7.3f}ms{pct}")


def main():
    ROUNDS = 3
    print(f"Redis benchmark ({DATA_SIZE}B values, {REQUESTS} reqs, {CLIENTS} clients, {ROUNDS} rounds)")
    print("=" * 85)

    all_results = {"bare": [], "sandlock": [], "docker": []}

    for r in range(1, ROUNDS + 1):
        print(f"\n--- Round {r}/{ROUNDS} ---")

        print("  Bare metal...", flush=True)
        res = bench_bare()
        all_results["bare"].append(res)
        print(f"    SET: {res['SET']['rps']:,.0f} rps  GET: {res['GET']['rps']:,.0f} rps")

        print("  Sandlock...", flush=True)
        res = bench_sandlock()
        all_results["sandlock"].append(res)
        print(f"    SET: {res['SET']['rps']:,.0f} rps  GET: {res['GET']['rps']:,.0f} rps")

        print("  Docker...", flush=True)
        res = bench_docker()
        all_results["docker"].append(res)
        print(f"    SET: {res['SET']['rps']:,.0f} rps  GET: {res['GET']['rps']:,.0f} rps")

    # Average results
    def avg_results(lst):
        out = {}
        for op in ("SET", "GET"):
            out[op] = {
                k: sum(r[op][k] for r in lst) / len(lst)
                for k in ("rps", "avg_ms", "p50_ms", "p99_ms")
            }
        return out

    print("\n" + "=" * 85)
    print(f"Average over {ROUNDS} rounds:")
    bare_avg = avg_results(all_results["bare"])
    sand_avg = avg_results(all_results["sandlock"])
    dock_avg = avg_results(all_results["docker"])

    print_header()
    print_row("Bare metal", bare_avg)
    print_row("Sandlock", sand_avg, bare_avg)
    print_row("Docker", dock_avg, bare_avg)


if __name__ == "__main__":
    main()