Cartesi Machine Sparse Computation Hash Benchmarks

bench-sparse.lua

local cartesi = require("cartesi")
local nanotime = require("chronos").nanotime

local benchs = {
  {name='rotate', desc='1 bit rotation stressor'},
  {name='matrix', desc='2D Matrix stressor'},
  {name='matrix-3d', desc='3D Matrix stressor'},
  {name='access', desc='Access stressor'},
  {name='affinity', desc='Affinity stressor'},
  {name='alarm', desc='Alarm stressor'},
  {name='memfd', desc='Anonymous file (memfd) stressor'},
  {name='aiol', desc='Asynchronous I/O stressor (Linux AIO)'},
  {name='aio', desc='Asynchronous I/O stressor (POSIX AIO)'},
  {name='atomic', desc='Atomic stressor'},
  {name='heapsort', desc='BSD heapsort stressor'},
  {name='mergesort', desc='BSD mergesort stressor'},
  {name='bad-altstack', desc='Bad alternative stack stressor'},
  {name='bigheap', desc='Big heap stressor'},
  {name='bsearch', desc='Binary search stressor'},
  {name='tsearch', desc='Binary tree stressor'},
  {name='bitonicsort', desc='Bitonic sort stressor'},
  {name='branch', desc='Branch stressor'},
  {name='brk', desc='Brk stressor'},
  {name='far-branch', desc='CPU branching instruction cache stressor'},
  {name='icache', desc='CPU instruction cache load stressor'},
  {name='goto', desc='CPU pipeline and branch prediction stressor'},
  {name='regs', desc='CPU registers stressor'},
  {name='cpu', desc='CPU stressor'},
  {name='syncload', desc='CPU synchronized loads stressor'},
  {name='waitcpu', desc='CPU wait instruction stressor'},
  {name='cacheline', desc='Cache line stressor'},
  {name='cache', desc='Cache stressor'},
  {name='chattr', desc='Chattr stressor'},
  {name='chdir', desc='Chdir stressor'},
  {name='chmod', desc='Chmod stressor'},
  {name='chown', desc='Chown stressor'},
  {name='clock', desc='Clock stressor'},
  {name='clone', desc='Clone stressor'},
  {name='close', desc='Close stressor'},
  {name='switch', desc='Context switching between mutually tied processes stressor'},
  {name='copy-file', desc='Copy file stressor'},
  {name='crypt', desc='Crypt stressor'},
  {name='daemon', desc='Daemon stressor'},
  {name='io', desc='Data synchronization (sync) stressor'},
  {name='dirdeep', desc='Deep directories stressor'},
  {name='dentry', desc='Dentry stressor'},
  {name='dir', desc='Directories stressor'},
  {name='stackmmap', desc='Dirty page and stack exception stressor'},
  {name='dnotify', desc='Dnotify stressor'},
  {name='dup', desc='Dup stressor'},
  {name='dynlib', desc='Dynamic libraries loading stressor'},
  {name='eigen', desc='Eigen C++ matrix library stressor'},
  {name='env', desc='Environment variables stressor'},
  {name='epoll', desc='Epoll stressor'},
  {name='eventfd', desc='Event file descriptor (eventfd) stressor'},
  {name='exec', desc='Exec processes stressor'},
  {name='exit-group', desc='Exiting pthread groups stressor'},
  {name='xattr', desc='Extended file attributes stressor'},
  {name='fifo', desc='FIFO named pipe stressor'},
  {name='factor', desc='Factorization of large integers stressor'},
  {name='futex', desc='Fast mutex (futex) stressor'},
  {name='schedmix', desc='Fast process rescheduling stressor'},
  {name='fcntl', desc='Fcntl stressor'},
  {name='get', desc='Fetching data from kernel stressor'},
  {name='file-ioctl', desc='File I/O control (ioctl) stressor'},
  {name='fd-fork', desc='File descriptor duplication and closing stressor'},
  {name='fiemap', desc='File extent (fiemap) stressor'},
  {name='lease', desc='File lease stressor'},
  {name='flock', desc='File locking (flock) stressor'},
  {name='metamix', desc='File metadata mix'},
  {name='fpunch', desc='File punch and hole filling stressor'},
  {name='seal', desc='File sealing (SEAL) stressor (Linux)'},
  {name='fsize', desc='File size limit stressor'},
  {name='fallocate', desc='File space allocation (fallocate) stressor'},
  {name='fstat', desc='File stats (fstat) stressor'},
  {name='utime', desc='File timestamp stressor'},
  {name='hdd', desc='File-system stressor'},
  {name='filename', desc='Filename stressor'},
  {name='mmapfixed', desc='Fixed address memory map stressor'},
  {name='fp-error', desc='Floating point exception stressor'},
  {name='fp', desc='Floating point operations stressor'},
  {name='mmapfork', desc='Forked memory map stressor'},
  {name='funccall', desc='Function argument passing stressor'},
  {name='funcret', desc='Function return stressor'},
  {name='fma', desc='Fused Multiply/Add floating point operations (fma) stressor'},
  {name='handle', desc='Handle stressor'},
  {name='link', desc='Hardlink stressor'},
  {name='hsearch', desc='Hashtable searching (hsearch) stressor'},
  {name='forkheavy', desc='Heavy process forking stressor'},
  {name='nanosleep', desc='High resolution and scheduler stressor via nanosleep calls'},
  {name='hrtimers', desc='High resolution timer stressor'},
  {name='mmaphuge', desc='Huge page memory mapping stressor'},
  {name='iomix', desc='IO mixing stressor'},
  {name='ioprio', desc='IO scheduling class and priority stressor'},
  {name='seek', desc='IO seek stressor'},
  {name='opcode', desc='Illegal instructions stressors'},
  {name='inode-flags', desc='Inode ioctl flags stressor'},
  {name='inotify', desc='Inotify stressor'},
  {name='io-uring', desc='Io-uring stressor'},
  {name='jpeg', desc='Jpeg compression stressor'},
  {name='judy', desc='Judy array stressor'},
  {name='kcmp', desc='Kcmp stressor (Linux)'},
  {name='prefetch', desc='L3 cache prefetching stressor'},
  {name='pipe', desc='Large pipe stressor'},
  {name='str', desc='Libc string functions stressor'},
  {name='wcs', desc='Libc wide characterstring function stressor'},
  {name='lsearch', desc='Linear search stressor'},
  {name='list', desc='List data structures stressor'},
  {name='loadavg', desc='Load average (loadavg) stressor'},
  {name='longjmp', desc='Long jump (longjmp) stressor'},
  {name='madvise', desc='Madvise stressor'},
  {name='mlock', desc='Mapped memory pages lock/unlock stressor'},
  {name='dirmany', desc='Maximum files creation in a directory stressor'},
  {name='mmapmany', desc='Maximum memory mapping per process stressor'},
  {name='vm', desc='Memory allocate and write stressor'},
  {name='malloc', desc='Memory allocation stressor'},
  {name='membarrier', desc='Memory barrier stressor (Linux)'},
  {name='mcontend', desc='Memory contention stressor'},
  {name='memcpy', desc='Memory copy (memcpy) stressor'},
  {name='mmapfiles', desc='Memory map files stressor'},
  {name='mmap', desc='Memory mapping (mmap/munmap) stressor'},
  {name='remap', desc='Memory page reordering stressor'},
  {name='pageswap', desc='Memory page swapping stressor'},
  {name='pkey', desc='Memory protection key mechanism stressor (Linux)'},
  {name='mprotect', desc='Memory protection stressor'},
  {name='memrate', desc='Memory read/write stressor'},
  {name='mremap', desc='Memory remap stressor (Linux)'},
  {name='memthrash', desc='Memory thrash stressor'},
  {name='vm-segv', desc='Memory unmap from a child process stressor'},
  {name='misaligned', desc='Misaligned read/write stressor'},
  {name='mknod', desc='Mknod/unlink stressor'},
  {name='monte-carlo', desc='Monte Carlo computations of <pi> and e and various integrals'},
  {name='mpfr', desc='Multi-precision floating operations (mpfr) stressor'},
  {name='dekker', desc='Mutex using Dekker algorithm stressor'},
  {name='peterson', desc='Mutex using Peterson algorithm stressor'},
  {name='nop', desc='NO-OP CPU instruction stressor'},
  {name='signest', desc='Nested signal handling stressor'},
  {name='netdev', desc='Network device ioctl stressor'},
  {name='sock', desc='Network socket stressor'},
  {name='nice', desc='Nice stressor'},
  {name='enosys', desc='Non-functional system call (ENOSYS) stressor'},
  {name='open', desc='Opening file (open) stressor'},
  {name='sockmany', desc='Opening network socket stressor'},
  {name='lockofd', desc='POSIX lock (F_OFD_SETLK/F_OFD_GETLK) stressor'},
  {name='locka', desc='POSIX lock (F_SETLK/F_GETLK) stressor'},
  {name='lockf', desc='POSIX lock (lockf) stressor'},
  {name='mq', desc='POSIX message queue stressor (Linux)'},
  {name='spawn', desc='POSIX process spawn (posix_spawn) stressor (Linux)'},
  {name='sem', desc='POSIX semaphore stressor'},
  {name='shm', desc='POSIX shared memory stressor'},
  {name='fault', desc='Page fault stressor'},
  {name='pagemove', desc='Page table and TLB stressor'},
  {name='sync-file', desc='Partial file syncing (sync_file_range) stressor'},
  {name='wait', desc='Pausing and resuming threads stressor'},
  {name='sigpending', desc='Pending signals stressor'},
  {name='personality', desc='Personality stressor'},
  {name='poll', desc='Polling stressor'},
  {name='prctl', desc='Prctl stressor'},
  {name='priv-instr', desc='Privileged CPU instructions stressor'},
  {name='cap', desc='Process capabilities stressor'},
  {name='fork', desc='Process forking stressor'},
  {name='resched', desc='Process rescheduling stressor'},
  {name='race-sched', desc='Process scheduler stressor'},
  {name='pidfd', desc='Process signals (pidfd_send_signal) stressor'},
  {name='kill', desc='Process signals stressor'},
  {name='pty', desc='Pseudo-terminals (pty) stressor'},
  {name='mutex', desc='Pthread mutex stressor'},
  {name='pthread', desc='Pthread stressor'},
  {name='ptrace', desc='Ptrace stressor'},
  {name='qsort', desc='Qsort stressor'},
  {name='radixsort', desc='Radixsort stressor'},
  {name='getrandom', desc='Random data (getrandom) stressor'},
  {name='randlist', desc='Random list stressor'},
  {name='skiplist', desc='Random memory and processor cache line stressor via a skiplist'},
  {name='mmapaddr', desc='Random memory map/unmap stressor'},
  {name='readahead', desc='Read-ahead stressor'},
  {name='sigrt', desc='Real-time signals stressor'},
  {name='reboot', desc='Reboot stressor'},
  {name='rename', desc='Renaming file stressor'},
  {name='mincore', desc='Resident memory (mincore) stressor'},
  {name='ring-pipe', desc='Ring pipes stressor'},
  {name='rlimit', desc='Rlimit stressor'},
  {name='sigabrt', desc='SIGABRT stressor'},
  {name='sigbus', desc='SIGBUS stressor'},
  {name='sigchld', desc='SIGCHLD stressor'},
  {name='sigfd', desc='SIGFD stressor (Linux)'},
  {name='sigfpe', desc='SIGFPE stressor'},
  {name='sigio', desc='SIGIO stressor'},
  {name='sigpipe', desc='SIGPIPE stressor'},
  {name='sigsegv', desc='SIGSEV stressor'},
  {name='sigtrap', desc='SIGTRAP stressor'},
  {name='sigxcpu', desc='SIGXCPU stressor'},
  {name='sigxfsz', desc='SIGXFSZ stressor'},
  {name='stream', desc='STREAM memory stressor'},
  {name='schedpolicy', desc='Scheduling policy stressor'},
  {name='secretmem', desc='Secret memory stressor (Linux >= 5.11)'},
  {name='sendfile', desc='Sendfile stressor'},
  {name='session', desc='Sessions stressor'},
  {name='set', desc='Setting data in the Kernel stressor'},
  {name='pipeherd', desc='Shared pipe stressor'},
  {name='shellsort', desc='Shellsort stressor'},
  {name='vecshuf', desc='Shuffled vector math operations stressor'},
  {name='sigq', desc='Signal queueing stressor'},
  {name='sockpair', desc='Socket I/O stressor'},
  {name='sockabuse', desc='Socket abusing stressor'},
  {name='sockdiag', desc='Socket diagnostic stressor (Linux)'},
  {name='sockfd', desc='Socket file descriptor stressor'},
  {name='sparsematrix', desc='Sparse matrix stressor'},
  {name='splice', desc='Splice stressor (Linux)'},
  {name='stack', desc='Stack stressor'},
  {name='hash', desc='String hashing stressor'},
  {name='context', desc='Swapcontext stressor'},
  {name='symlink', desc='Symlink stressor'},
  {name='msyncmany', desc='Synchronize file with memory map (msync) coherency stressor'},
  {name='msync', desc='Synchronize file with memory map (msync) stressor'},
  {name='klog', desc='Syslog stressor (Linux)'},
  {name='msg', desc='System V message IPC stressor'},
  {name='sem-sysv', desc='System V semaphore stressor'},
  {name='sysbadaddr', desc='System calls bad address and fault handling stressor'},
  {name='syscall', desc='System calls stressor'},
  {name='sysinfo', desc='System information stressor'},
  {name='itimer', desc='System interval timer stressor'},
  {name='resources', desc='System resources stressor'},
  {name='signal', desc='System signals stressor'},
  {name='tee', desc='Tee stressor (Linux)'},
  {name='sleep', desc='Time interrupts and context switches stressor'},
  {name='tsc', desc='Time stamp counter (TSC) stressor'},
  {name='time-warp', desc='Time warp stressor'},
  {name='timer', desc='Timer event stressor (Linux)'},
  {name='timerfd', desc='Timerfd stressor (Linux)'},
  {name='touch', desc='Touching files stressor'},
  {name='tlb-shootdown', desc='Translation lookaside buffer shootdowns stressor'},
  {name='tree', desc='Tree data structures stressor'},
  {name='trig', desc='Trigonometric functions stressor'},
  {name='udp-flood', desc='UDP flooding stressor'},
  {name='udp', desc='UDP network stressor'},
  {name='munmap', desc='Unmapping shared non-executable memory stressor (Linux)'},
  {name='unshare', desc='Unshare stressor (Linux)'},
  {name='rmap', desc='VM reverse-mapping stressor'},
  {name='vecfp', desc='Vector floating point operations stressor'},
  {name='vecmath', desc='Vector math operations stressor'},
  {name='vnni', desc='Vector neural network instructions stressor'},
  {name='vma', desc='Virtual Memory Area (VMA) stressor'},
  {name='vdso', desc='Virtual dynamic shared object stressor'},
  {name='getdent', desc='Virtual filesystem directories stressor (Linux)'},
  {name='vm-addr', desc='Virtual memory addressing stressor'},
  {name='vm-splice', desc='Vmsplice stressor (Linux)'},
  {name='sigsuspend', desc='Waiting for process signals stressor'},
  {name='vecwide', desc='Wide vector math operations stressor'},
  {name='revio', desc='Writing temporary files in reverse position stressor'},
  {name='yield', desc='Yield scheduling stressor'},
  {name='zlib', desc='Zlib stressor'},
  {name='zombie', desc='Zombie processes stressor'},

  -- Require root
  {name='cgroup', desc='Cgroup stressor', asroot=true},
  {name='chroot', desc='Chroot stressor', asroot=true},
  {name='icmp-flood', desc='ICMP flooding stressor', asroot=true},
  {name='ramfs', desc='Memory filesystem stressor', asroot=true},
  {name='rawpkt', desc='Localhost ethernet raw packets stressor', asroot=true},
  {name='rawsock', desc='Localhost raw socket stressor', asroot=true},
  {name='rawudp', desc='Localhost raw UDP packet stressor', asroot=true},
  {name='seccomp', desc='Secure computing stressor', asroot=true},
  {name='softlockup', desc='Softlockup stressor', asroot=true},
  {name='tun', desc='Network tunnel stressor', asroot=true},
  {name='umount', desc='Umount stressor', asroot=true},

  -- Unsupported
  -- {name='acl', desc='POSIX Access Control List Stressor'},
  -- {name='af-alg', desc='Kernel crypto AF_ALG API stressor'},
  -- {name='apparmor', desc='AppArmor stressor'},
  -- {name='bad-ioctl', desc='Bad ioctl stressor'},
  -- {name='bind-mount', desc='Bind mount stressor'},
  -- {name='binderfs', desc='Binderfs stressor'},
  -- {name='cpu-online', desc='CPU onlining stressor'},
  -- {name='cyclic', desc='Cyclic stressor'},
  -- {name='dccp', desc='Datagram congestion control protocol (DCCP) stressor'},
  -- {name='efivar', desc='EFI variables stressor'},
  -- {name='fanotify', desc='Filesystem notification (fanotify) stressor'},
  -- {name='flush-cache', desc='Cache flushing stressor'},
  -- {name='gpu', desc='2D GPU stressor'},
  -- {name='idle-scan', desc='Idle pages stressor (Linux)'},
  -- {name='ioport', desc='Ioport stressor (x86 Linux)'},
  -- {name='ipsec-mb', desc='Ipsec multi-buffer cryptographic stressor'},
  -- {name='key', desc='Kernel key management stressor'},
  -- {name='kvm', desc='KVM stressor'},
  -- {name='l1cache', desc='CPU L1 cache stressor'},
  -- {name='landlock', desc='Landlock stressor (Linux >= 5.13)'},
  -- {name='led', desc='LED stressor (Linux)'},
  -- {name='llc-affinity', desc='Last level of cache stressor'},
  -- {name='lockbus', desc='Lock and increment memory stressor (x86 and ARM)'},
  -- {name='loop', desc='Loopback stressor (Linux)'},
  -- {name='memhotplug', desc='Memory hotplug stressor (Linux)'},
  -- {name='module', desc='Kernel module loading stressor (Linux)'},
  -- {name='netlink-proc', desc='Netlink stressor (Linux)'},
  -- {name='numa', desc='Migrate memory pages over NUMA nodes stressor'},
  -- {name='oom-pipe', desc='Large Pipe stressor'},
  -- {name='pci', desc='PCI sysfs stressor (Linux)'},
  -- {name='physpage', desc='Page map stressor'},
  -- {name='ping-sock', desc='Localhost ICMP (ping) stressor'},
  -- {name='plugin', desc='Stress-ng plugin stressor'},
  -- {name='quota', desc='Quota stressor'},
  -- {name='rawdev', desc='Raw device stressor'},
  -- {name='rdrand', desc='Random number generator stressor'},
  -- {name='rseq', desc='Restartable sequences (rseq) stressor (Linux)'},
  -- {name='rtc', desc='Real-time clock stressor'},
  -- {name='sctp', desc='Stream control transmission protocol (SCTP) stressor'},
  -- {name='smi', desc='System management interrupts (SMI) stressor'},
  -- {name='swap', desc='Swap partitions stressor (Linux)'},
  -- {name='sysinval', desc='System calls with invalid arguments stressor (Linux)'},
  -- {name='tmpfs', desc='Tmpfs stressor'},
  -- {name='uprobe', desc='Uprobe stressor (Linux)'},
  -- {name='userfaultfd', desc='Page faults stressor (Linux)'},
  -- {name='usersyscall', desc='SYGSYS stressor'},
  -- {name='verity', desc='File based authenticy protection (verity) stressor'},
  -- {name='vm-rw', desc='Memory transfer between parent and child processes stressor (Linux)'},
  -- {name='watchdog', desc='Watchdog stressor'},
}

-- These benchmarks were selected to stress different characteristics of memory/cpu
local benchs = {
  {name='nop', desc='NO-OP CPU instruction stressor'},
  {name='crypt', desc='Crypt stressor'},
  {name='heapsort', desc='BSD heapsort stressor'},
  {name='tsearch', desc='Binary tree stressor'},
  {name='memthrash', desc='Memory thrash stressor'},
  {name='matrix-3d', desc='3D Matrix stressor'},
  {name='tree', desc='Tree data structures stressor'},
  {name='tlb-shootdown', desc='Translation lookaside buffer shootdowns stressor'},
  {name='malloc', desc='Memory allocation stressor'},
  {name='randlist', desc='Random list stressor'},
}

local function numstr_i(n)
  if n >= 1024*1024*1024 then return n//(1024*1024*1024) .. 'Gi' end
  if n >= 1024*1024 then return n//(1024*1024) .. 'Mi' end
  if n >= 1024 then return n//1024 .. 'Ki' end
  return n
end

local function numstr(n)
  if n >= 1000*1000 then return string.format('%.2fM', n/(1000*1000)) end
  if n >= 1000 then return string.format('%.2fK', n/1000) end
  return n
end

local images_dir = os.getenv('CARTESI_IMAGES_PATH')
local boot_mcycle = 256*1024*1024 -- enough to skip Linux boot and get in the middle of a benchmark
local ram_length = 512*1024*1024 -- 512MiB
local max_step_log2 = 30 -- 1Gi mcycles
local min_step_log2 = 13 -- 8Ki mcycles
local compute_mcycles = 1 << max_step_log2
local max_mcycle = boot_mcycle + compute_mcycles

print('| benchmark        | period | speed      | slowdown | word hit | page hit | pristine hit | change hit | page changes | page hashes | dense hashes | sparse hashes |')
print('|------------------|--------|------------|----------|----------|----------|--------------|------------|--------------|-------------|--------------|---------------|')

for _,bench in ipairs(benchs) do
  local config = {
    ram = {
        length = ram_length,
        backing_store = {
            data_filename = images_dir .. "/linux.bin",
            truncate = true,
        },
    },
    dtb = {
        init = bench.asroot and "USER=root" or '',
        entrypoint = "stress-ng --no-rand-seed --"..bench.name.." 1 --timeout 1m >> /dev/null",
    },
    flash_drive = {
        {
            backing_store = {
                data_filename = images_dir .. "/rootfs.ext2",
            },
        },
    },
  }
  local first_elapsed
  local first_final_hash
  for step_log2=max_step_log2,min_step_log2,-1 do
    local machine <close> = cartesi.machine(config)
    machine:get_root_hash()
    machine:run(boot_mcycle)
    -- get the root hash twice, the second time will make always-dirty pages into the page cache
    machine:get_root_hash()
    machine:get_root_hash()
    machine:get_hash_tree_stats(true)
    local step = 1 << step_log2
    local start = nanotime()
    local periods = 0
    for mcycle=boot_mcycle+step,boot_mcycle+compute_mcycles,step do
      assert(machine:run(mcycle) == cartesi.BREAK_REASON_REACHED_TARGET_MCYCLE, 'benchmark ended too early, increase its complexity to fi')
      machine:get_root_hash()
      periods = periods + 1
    end
    local elapsed = nanotime() - start
    if not first_elapsed then
      first_elapsed = elapsed
      first_final_hash = machine:get_root_hash()
    else
      assert(first_final_hash == machine:get_root_hash(), 'final hash mismatch')
    end
    local relative_elapsed = elapsed / first_elapsed
    local stats = machine:get_hash_tree_stats(true)
    local page_stats = stats.phtc
    local page_hits = page_stats.page_hits * 100.0 / (page_stats.page_hits + page_stats.page_misses)
    local dense_node_hashes = 0
    for level,hashes in pairs(stats.dense_node_hashes) do
      dense_node_hashes = dense_node_hashes + hashes
    end
    local speed = compute_mcycles / elapsed
    local period_page_hashes = (page_stats.inner_page_hashes + page_stats.word_misses) / periods
    local period_sparse_hashes = stats.sparse_node_hashes / periods
    local period_dense_hashes = dense_node_hashes / periods
    local period_page_changes = page_stats.page_changes / periods
    local page_checks = page_stats.pristine_pages + page_stats.non_pristine_pages
    local word_checks = page_stats.word_hits + page_stats.word_misses
    assert(page_stats.inner_page_hashes + page_stats.word_misses <= page_checks*255)
    local pristine_hits = page_stats.pristine_pages * 100.0 / page_checks
    local change_hit = page_stats.page_changes * 100 / page_checks
    local word_hits = page_stats.word_hits * 100.0 / word_checks
    print(string.format('| %-16s | %6s | %8sHz | %8.2f | %7.3f%% | %7.3f%% | %11.7f%% | %9.3f%% | %12.2f | %11.2f | %12.2f | %13.2f |', bench.name, numstr_i(step), numstr(speed), relative_elapsed, word_hits, page_hits, pristine_hits, change_hit, period_page_changes, period_page_hashes, period_dense_hashes, period_sparse_hashes))
  end
end