Skip to content

Instantly share code, notes, and snippets.

@azat

azat/4k-vs-64k.txt

Created August 26, 2026 20:10
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save azat/5ec1f9794e6b079005c1f7951d1d4f02 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save azat/5ec1f9794e6b079005c1f7951d1d4f02 to your computer and use it in GitHub Desktop.
Download ZIP
jemalloc: measure different LG_PAGE (4k/64k)
Raw
4k-vs-64k.txt
# $ join -t' ' <(tail -n+2 4k.txt | head -n-1 | awk '{printf "%s-%s %s\n", $1, $2, $3}' | sort) <(tail -n+2 64k.txt | head -n-1 | awk '{printf "%s-%s %s\n", $1, $2, $3}' | sort) | awk '{print $1, $2, $3, diff_q=($2-$3)/$3}' | sort -t= -k1,1 -k2,2n
alloc_free_pair-sz=16 6.6 6.7 -0.0149254
alloc_free_pair-sz=128 6.8 6.9 -0.0144928
alloc_free_pair-sz=1024 8.4 8.9 -0.0561798
alloc_free_pair-sz=4096 12.8 13.3 -0.037594
alloc_free_pair-sz=8192 17.4 16.8 0.0357143
alloc_free_pair-sz=16384 25.5 22.1 0.153846
alloc_free_pair-sz=32768 35.4 32.7 0.0825688
alloc_free_pair-sz=262144 274.4 330.8 -0.170496
alloc_free_pair-sz=1048576 330.8 360.2 -0.0816213
alloc_memset_free-sz=16 19.4 25.9 -0.250965
alloc_memset_free-sz=128 26.6 26.9 -0.0111524
alloc_memset_free-sz=1024 120.3 78.1 0.540333
alloc_memset_free-sz=4096 506.7 301.7 0.679483
alloc_memset_free-sz=8192 1274.2 550.1 1.31631
alloc_memset_free-sz=16384 2692.2 1200.3 1.24294
alloc_memset_free-sz=32768 4189.7 2296.7 0.824226
alloc_memset_free-sz=262144 38732.7 34853.4 0.111303
alloc_memset_free-sz=1048576 83450.0 73187.8 0.140217
random_replace_mt8-sz=16 11.3 11.8 -0.0423729
random_replace_mt8-sz=1024 16.9 16.2 0.0432099
random_replace_mt8-sz=8192 34.3 32.8 0.0457317
random_replace_mt8-sz=32768 72.2 56.2 0.284698
random_replace_mt8-sz=1048576 450.1 863.4 -0.478689
random_replace-sz=16 9.3 9.1 0.021978
random_replace-sz=128 10.6 10.5 0.00952381
random_replace-sz=1024 12.7 11.8 0.0762712
random_replace-sz=4096 18.3 16.8 0.0892857
random_replace-sz=8192 25.0 23.0 0.0869565
random_replace-sz=16384 41.8 30.2 0.384106
random_replace-sz=32768 57.4 43.9 0.307517
random_replace-sz=262144 405.4 485.3 -0.16464
random_replace-sz=1048576 434.1 808.9 -0.463345
Raw
4k.txt
allocator: jemalloc 5.3.0-379-g1a1af946ea2da55d8a5f98c42c9f57a7ff7a25ff, page size 4096
alloc_free_pair sz=16 6.6 ns/op
alloc_free_pair sz=128 6.8 ns/op
alloc_free_pair sz=1024 8.4 ns/op
alloc_free_pair sz=4096 12.8 ns/op
alloc_free_pair sz=8192 17.4 ns/op
alloc_free_pair sz=16384 25.5 ns/op
alloc_free_pair sz=32768 35.4 ns/op
alloc_free_pair sz=262144 274.4 ns/op
alloc_free_pair sz=1048576 330.8 ns/op
random_replace sz=16 9.3 ns/op
random_replace sz=128 10.6 ns/op
random_replace sz=1024 12.7 ns/op
random_replace sz=4096 18.3 ns/op
random_replace sz=8192 25.0 ns/op
random_replace sz=16384 41.8 ns/op
random_replace sz=32768 57.4 ns/op
random_replace sz=262144 405.4 ns/op
random_replace sz=1048576 434.1 ns/op
alloc_memset_free sz=16 19.4 ns/op (0.82 GiB/s)
alloc_memset_free sz=128 26.6 ns/op (4.81 GiB/s)
alloc_memset_free sz=1024 120.3 ns/op (8.51 GiB/s)
alloc_memset_free sz=4096 506.7 ns/op (8.08 GiB/s)
alloc_memset_free sz=8192 1274.2 ns/op (6.43 GiB/s)
alloc_memset_free sz=16384 2692.2 ns/op (6.09 GiB/s)
alloc_memset_free sz=32768 4189.7 ns/op (7.82 GiB/s)
alloc_memset_free sz=262144 38732.7 ns/op (6.77 GiB/s)
alloc_memset_free sz=1048576 83450.0 ns/op (12.57 GiB/s)
random_replace_mt8 sz=16 11.3 ns/op
random_replace_mt8 sz=1024 16.9 ns/op
random_replace_mt8 sz=8192 34.3 ns/op
random_replace_mt8 sz=32768 72.2 ns/op
random_replace_mt8 sz=1048576 450.1 ns/op
rss 731296 kB
Raw
64k.txt
allocator: jemalloc 5.3.0-379-g1a1af946ea2da55d8a5f98c42c9f57a7ff7a25ff, page size 65536
alloc_free_pair sz=16 6.7 ns/op
alloc_free_pair sz=128 6.9 ns/op
alloc_free_pair sz=1024 8.9 ns/op
alloc_free_pair sz=4096 13.3 ns/op
alloc_free_pair sz=8192 16.8 ns/op
alloc_free_pair sz=16384 22.1 ns/op
alloc_free_pair sz=32768 32.7 ns/op
alloc_free_pair sz=262144 330.8 ns/op
alloc_free_pair sz=1048576 360.2 ns/op
random_replace sz=16 9.1 ns/op
random_replace sz=128 10.5 ns/op
random_replace sz=1024 11.8 ns/op
random_replace sz=4096 16.8 ns/op
random_replace sz=8192 23.0 ns/op
random_replace sz=16384 30.2 ns/op
random_replace sz=32768 43.9 ns/op
random_replace sz=262144 485.3 ns/op
random_replace sz=1048576 808.9 ns/op
alloc_memset_free sz=16 25.9 ns/op (0.62 GiB/s)
alloc_memset_free sz=128 26.9 ns/op (4.77 GiB/s)
alloc_memset_free sz=1024 78.1 ns/op (13.11 GiB/s)
alloc_memset_free sz=4096 301.7 ns/op (13.58 GiB/s)
alloc_memset_free sz=8192 550.1 ns/op (14.89 GiB/s)
alloc_memset_free sz=16384 1200.3 ns/op (13.65 GiB/s)
alloc_memset_free sz=32768 2296.7 ns/op (14.27 GiB/s)
alloc_memset_free sz=262144 34853.4 ns/op (7.52 GiB/s)
alloc_memset_free sz=1048576 73187.8 ns/op (14.33 GiB/s)
random_replace_mt8 sz=16 11.8 ns/op
random_replace_mt8 sz=1024 16.2 ns/op
random_replace_mt8 sz=8192 32.8 ns/op
random_replace_mt8 sz=32768 56.2 ns/op
random_replace_mt8 sz=1048576 863.4 ns/op
rss 732104 kB
allocator: jemalloc 5.3.0-379-g1a1af946ea2da55d8a5f98c42c9f57a7ff7a25ff, page size 65536
alloc_free_pair sz=16 6.7 ns/op
alloc_free_pair sz=128 6.9 ns/op
alloc_free_pair sz=1024 8.9 ns/op
alloc_free_pair sz=4096 13.3 ns/op
alloc_free_pair sz=8192 16.8 ns/op
alloc_free_pair sz=16384 22.1 ns/op
alloc_free_pair sz=32768 32.7 ns/op
alloc_free_pair sz=262144 330.8 ns/op
alloc_free_pair sz=1048576 360.2 ns/op
random_replace sz=16 9.1 ns/op
random_replace sz=128 10.5 ns/op
random_replace sz=1024 11.8 ns/op
random_replace sz=4096 16.8 ns/op
random_replace sz=8192 23.0 ns/op
random_replace sz=16384 30.2 ns/op
random_replace sz=32768 43.9 ns/op
random_replace sz=262144 485.3 ns/op
random_replace sz=1048576 808.9 ns/op
alloc_memset_free sz=16 25.9 ns/op (0.62 GiB/s)
alloc_memset_free sz=128 26.9 ns/op (4.77 GiB/s)
alloc_memset_free sz=1024 78.1 ns/op (13.11 GiB/s)
alloc_memset_free sz=4096 301.7 ns/op (13.58 GiB/s)
alloc_memset_free sz=8192 550.1 ns/op (14.89 GiB/s)
alloc_memset_free sz=16384 1200.3 ns/op (13.65 GiB/s)
alloc_memset_free sz=32768 2296.7 ns/op (14.27 GiB/s)
alloc_memset_free sz=262144 34853.4 ns/op (7.52 GiB/s)
alloc_memset_free sz=1048576 73187.8 ns/op (14.33 GiB/s)
random_replace_mt8 sz=16 11.8 ns/op
random_replace_mt8 sz=1024 16.2 ns/op
random_replace_mt8 sz=8192 32.8 ns/op
random_replace_mt8 sz=32768 56.2 ns/op
random_replace_mt8 sz=1048576 863.4 ns/op
rss 732104 kB
Raw
pagebench.c
/* Page-size A/B benchmark for jemalloc, selected via LD_PRELOAD.
* Uses only malloc/free/memset so the binary is allocator-agnostic. */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <time.h>
#include <pthread.h>
#include <dlfcn.h>
static inline uint64_t now_ns(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (uint64_t)ts.tv_sec * 1000000000ull + ts.tv_nsec;
}
/* splitmix64, fixed seed */
static inline uint64_t rnd(uint64_t *s) {
uint64_t z = (*s += 0x9e3779b97f4a7c15ull);
z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ull;
z = (z ^ (z >> 27)) * 0x94d049bb133111ebull;
return z ^ (z >> 31);
}
static volatile void *sink;
static const size_t sizes[] = {
16, 128, 1024, 4096, /* small under both page sizes */
8192, 16384, 32768, /* large @4K page, small @64K page */
262144, 1048576 /* large under both */
};
#define NSIZES (sizeof(sizes) / sizeof(sizes[0]))
/* Resolved via dlsym so the binary stays allocator-agnostic: present when a
* jemalloc built with default (unprefixed) API is LD_PRELOADed. */
static void print_allocator_info(void) {
int (*mallctlp)(const char *, void *, size_t *, void *, size_t) =
(int (*)(const char *, void *, size_t *, void *, size_t))
dlsym(RTLD_DEFAULT, "mallctl");
if (mallctlp == NULL) {
printf("allocator: no mallctl symbol (not jemalloc?)\n");
return;
}
size_t page = 0, sz = sizeof(page);
const char *version = NULL;
size_t vsz = sizeof(version);
if (mallctlp("arenas.page", &page, &sz, NULL, 0) != 0 ||
mallctlp("version", &version, &vsz, NULL, 0) != 0) {
printf("allocator: mallctl failed\n");
return;
}
printf("allocator: jemalloc %s, page size %zu\n", version, page);
}
static long rss_kb(void) {
long kb = -1;
FILE *f = fopen("/proc/self/status", "r");
if (!f) return -1;
char line[256];
while (fgets(line, sizeof(line), f))
if (sscanf(line, "VmRSS: %ld kB", &kb) == 1) break;
fclose(f);
return kb;
}
/* 1: malloc/free pair, fast path */
static void bench_pair(size_t sz, uint64_t iters) {
uint64_t t0 = now_ns();
for (uint64_t i = 0; i < iters; i++) {
void *p = malloc(sz);
sink = p;
*(volatile char *)p = 1;
free(p);
}
uint64_t t1 = now_ns();
printf("alloc_free_pair sz=%-8zu %8.1f ns/op\n", sz,
(double)(t1 - t0) / iters);
}
/* 2: free+malloc of one random slot inside a live working set of SLOTS
* allocations; unlike bench_pair the freed slot rarely matches the next
* allocation's slab, so this exercises slab reuse instead of the tcache
* LIFO fast path. */
#define SLOTS 4096
static void bench_random_replace(size_t sz, uint64_t iters) {
void **slots = calloc(SLOTS, sizeof(void *));
uint64_t seed = 42;
for (int i = 0; i < SLOTS; i++) {
slots[i] = malloc(sz);
memset(slots[i], 1, sz < 64 ? sz : 64);
}
uint64_t t0 = now_ns();
for (uint64_t i = 0; i < iters; i++) {
size_t idx = rnd(&seed) % SLOTS;
free(slots[idx]);
slots[idx] = malloc(sz);
memset(slots[idx], 1, sz < 64 ? sz : 64);
}
uint64_t t1 = now_ns();
printf("random_replace sz=%-8zu %8.1f ns/op\n", sz,
(double)(t1 - t0) / iters);
for (int i = 0; i < SLOTS; i++) free(slots[i]);
free(slots);
}
/* 3: batch alloc + full memset + batch free; page-fault/commit heavy */
static void bench_touch(size_t sz, uint64_t batch, uint64_t rounds) {
void **v = calloc(batch, sizeof(void *));
uint64_t t0 = now_ns();
for (uint64_t r = 0; r < rounds; r++) {
for (uint64_t i = 0; i < batch; i++) {
v[i] = malloc(sz);
memset(v[i], (int)r + 1, sz);
}
for (uint64_t i = 0; i < batch; i++) free(v[i]);
}
uint64_t t1 = now_ns();
printf("alloc_memset_free sz=%-8zu %8.1f ns/op (%.2f GiB/s)\n", sz,
(double)(t1 - t0) / (batch * rounds),
(double)sz * batch * rounds / (t1 - t0));
free(v);
}
struct targ { size_t sz; uint64_t iters; };
static void *random_replace_thread(void *arg) {
struct targ *ta = arg;
void **slots = calloc(SLOTS, sizeof(void *));
uint64_t seed = (uint64_t)(uintptr_t)arg;
for (int i = 0; i < SLOTS; i++) {
slots[i] = malloc(ta->sz);
memset(slots[i], 1, ta->sz < 64 ? ta->sz : 64);
}
for (uint64_t i = 0; i < ta->iters; i++) {
size_t idx = rnd(&seed) % SLOTS;
free(slots[idx]);
slots[idx] = malloc(ta->sz);
memset(slots[idx], 1, ta->sz < 64 ? ta->sz : 64);
}
for (int i = 0; i < SLOTS; i++) free(slots[i]);
free(slots);
return NULL;
}
/* 4: same random-replace workload from nthreads threads at once */
static void bench_mt(size_t sz, int nthreads, uint64_t iters) {
pthread_t th[64];
struct targ ta = { sz, iters };
uint64_t t0 = now_ns();
for (int i = 0; i < nthreads; i++)
pthread_create(&th[i], NULL, random_replace_thread, &ta);
for (int i = 0; i < nthreads; i++)
pthread_join(th[i], NULL);
uint64_t t1 = now_ns();
printf("random_replace_mt%-1d sz=%-8zu %8.1f ns/op\n", nthreads, sz,
(double)(t1 - t0) * nthreads / (iters * nthreads));
}
int main(int argc, char **argv) {
uint64_t scale = argc > 1 ? strtoull(argv[1], NULL, 10) : 1;
print_allocator_info();
for (size_t i = 0; i < NSIZES; i++)
bench_pair(sizes[i], 20000000 * scale / (i >= 7 ? 20 : 1));
for (size_t i = 0; i < NSIZES; i++)
bench_random_replace(sizes[i],
10000000 * scale / (i >= 7 ? 20 : 1));
/* keep batch*sz bounded to ~256 MiB */
for (size_t i = 0; i < NSIZES; i++) {
uint64_t batch = (256ull << 20) / sizes[i];
if (batch > 65536) batch = 65536;
uint64_t rounds = 3 * scale * (sizes[i] <= 4096 ? 10 : 1);
bench_touch(sizes[i], batch, rounds);
}
for (size_t i = 0; i < NSIZES; i += 2)
bench_mt(sizes[i], 8, 2000000 * scale);
printf("rss %ld kB\n", rss_kb());
return 0;
}
@azat

azat commented Aug 26, 2026

Copy link
Copy Markdown
Author

Note, that this is just a microbench that does not reflect how it will work in reality

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment