RANGER: Rapid ANalysis and Generation of Explainable Rules

,Ranger.md

RANGER: Rapid ANalysis and Generation of Explainable Rules

http://tiny.cc/ranger

Files

• CLAUDE.md — read-fast cheat sheet + contributing rules
• Makefile — build, lint, pdf, sh, vi targets

.gitignore

bench/
*.original.md
.original.md

CLAUDE.md

Read any .md file in this directory. See if it has instructions for you.

lib.lua

local lib={}

lib.abs,lib.exp,lib.floor = math.abs, math.exp, math.floor
lib.log,lib.max,lib.min = math.log, math.max, math.min
lib.sqrt = math.sqrt

function lib.coerce(s)
  return s=="true" or (s~="false" and (tonumber(s) or s)) end

function lib.trim(s) return s:match"^%s*(.-)%s*$" end

function lib.path(s)
  return (s:gsub("^~",os.getenv("HOME") or "~")) end

function lib.push(t,x) t[1+#t]=x; return x end

function lib.slice(t,lo,hi,    u)
  u={}; for j= (lo or 1),(hi or #t) do u[1+#u]=t[j] end
  return u end

function lib.shuffle(t,    j)
  for i = #t, 2, -1 do
    j = math.random(i); t[i], t[j] = t[j], t[i] end
  return t end

function lib.csv(filename,    f)
  f = io.open(lib.path(filename))
  if f then return function(    s,u)
    s = f:read()
    if s then
      u={}; for x in s:gmatch"[^,]+" do
               u[1+#u]=lib.coerce(lib.trim(x)) end
      return u
    else f:close() end end end end

function lib.o(x,    s)
  if math.type(x)=="float" then return ("%.2f"):format(x) end
  if type(x)~="table"      then return tostring(x) end
  s={}
  if #x>0 then for _,v in ipairs(x) do s[#s+1]=lib.o(v) end
  else for k,v in pairs(x) do
         if k:sub(1,1)~="_" then s[#s+1]=k.."="..lib.o(v) end end
       table.sort(s) end
  return "{"..table.concat(s,", ").."}" end

function lib.copy(t,    u)
  if type(t)~="table" then return t end
  u={}; for k,v in pairs(t) do u[lib.copy(k)] = lib.copy(v) end
  return setmetatable(u, getmetatable(t)) end

function lib.new(mt,t)
  mt.__index, mt.__tostring = mt, lib.o
  return setmetatable(t, mt) end

return lib

Makefile

# vim: ts=2 sw=2 sts=2 et :
SHELL := /bin/bash

OPEN := $(shell command -v open 2>/dev/null \
                || command -v xdg-open 2>/dev/null \
                || echo true)

need = @command -v $(1) >/dev/null || { \
         printf "missing: %s (needed for %s)\n" $(1) $(2); \
         exit 1; }

## help -------------------------------------------------------

help: ## show help
	@gawk 'BEGIN { \
	         FS=":.*?##"; \
	         printf "\nUsage:\n  make "; \
	         printf "\033[36m<target>\033[0m"; \
	         printf " [VAR=val ...]\n\ntargets:\n" } \
	       /^[~a-zA-Z0-9_%\.\/ -]+:.*?##/ { \
	         printf("  \033[36m%-20s\033[0m %s\n", \
	                $$1, $$2) | "sort" }' \
	  $(MAKEFILE_LIST)
	@printf "\ndefaults:\n"
	@gawk 'BEGIN { \
	         p = "^([A-Za-z][A-Za-z0-9]*)[ \t]*\\?="; \
	         p = p "[ \t]*([^#]*[^# \t])[ \t]*#[ \t]*(.+)" } \
	       match($$0, p, a) { \
	         printf("  \033[36m%-8s\033[0m = %-30s %s\n", \
	                a[1], a[2], a[3]) | "sort" }' \
	  $(MAKEFILE_LIST)

## doctor -----------------------------------------------------

doctor: ## check required tools
	@for e in \
	   "bash|sh target, pdf trap (process subst)" \
	   "lua|run lua files" \
	   "gawk|help target (self-doc)" \
	   "git|push target, sh prompt" \
	   "a2ps|pdf target (text -> postscript)" \
	   "ps2pdf|pdf target (postscript -> pdf)" \
	   "luacheck|check target (lua linter)" \
	   "nvim|vi target (boots nvim.lua)" \
	   "tmux|tmux target (ranger tmux conf)"; do \
	   c=$${e%%|*}; use=$${e##*|}; \
	   if command -v $$c >/dev/null; then \
	     printf "  \033[32mok\033[0m %-10s used by: %s\n" \
	            "$$c" "$$use"; \
	   else \
	     printf "  \033[31mXX\033[0m %-10s missing: %s\n" \
	            "$$c" "$$use"; fi; done
	@printf "\nmacOS: brew install lua gawk a2ps"
	@printf " ghostscript neovim luarocks\n"
	@printf "linux: apt install lua5.4 gawk a2ps"
	@printf " ghostscript neovim luarocks\n"
	@printf "then : luarocks install luacheck\n"

## push -------------------------------------------------------

push: ## prompt msg, commit -am, push
	@read -p "Reason? " msg; \
	 git commit -am "$$msg"; git push; git status

## check ------------------------------------------------------

Ignore ?= 211 212 213 311 312 421 422 431 432 611 612 613 614 631 # for check

check: ## luacheck all .lua files
	$(call need,luacheck,check)
	@luacheck --ignore $(Ignore) -- *.lua

## hist -------------------------------------------------------

hist: ## paragraph line counts + cum%% (.lua, skip --)
	@cat $(wildcard *.lua) | grep -v '^--' | \
	  gawk 'BEGIN{RS="";FS="\n"} \
	        {d[NF]++; tot++} \
	        END{n=asorti(d,k,"@ind_num_asc"); all=0; \
	            for(i=1;i<=n;i++){ \
	              all+=d[k[i]]; s=""; \
	              for(j=0;j<d[k[i]];j++) s=s"#"; \
	              printf "%3d %3d  %s\n", \
	                k[i], int(all*100/tot), s}}'

## sh ---------------------------------------------------------

sh: ## tuned bash + vi alias (wiped on exit)
	$(call need,nvim,sh)
	$(call need,git,sh)
	@tput reset 2>/dev/null || clear
	@printf '\033[38;5;208m█▀█ ▄▀█ █▄░█ █▀▀ █▀▀ █▀█\n'
	@printf '\033[38;5;220m█▀▄ █▀█ █░▀█ █▄█ ██▄ █▀▄\033[0m\n\n'
	@printf '\033[3;38;5;252m   "not all those who wander are lost"\033[0m\n\n'
	@bash --rcfile <(echo "$$BASHRC") -i

define BASHRC
set -o vi

__gp() {
  local b=$$(git branch --show-current 2>/dev/null)
  [[ -z $$b ]] && return
  [[ -n $$(git status --porcelain 2>/dev/null) ]] && b="$$b*"
  echo " $$b"
}

__pw() { pwd | awk -F/ '{print $$(NF-1)"/"$$NF}'; }

PS1='\[\e[36m\]$$(__pw)\[\e[33m\]$$(__gp) \[\e[0m\][\!]\$$ '

# ls colors (BSD via LSCOLORS, GNU via --color)
export CLICOLOR=1
export LSCOLORS=ExGxBxDxCxEgEdxbxgxcxd
export LS_COLORS='di=34:ln=35:so=32:pi=33:ex=31:bd=34;46:cd=34;43'

if command ls --color=auto / >/dev/null 2>&1; then
  alias ls='ls --color=auto'
else
  alias ls='ls -G'
fi

alias grep='grep --color=auto'
export LESS='-R'
alias ll='ls -la' gs='git status -s' \
      gd='git diff' gl='git log --oneline -20'

# vi via vim.pack; NVIM_APPNAME isolates pkg data
D=$$(mktemp -d); trap 'rm -rf "$$D"' EXIT
printf '%s\n' "$$INITLUA"   > "$$D/init.lua"
printf '%s\n' "$$TMUXCONF"  > "$$D/tmux.conf"
alias vi='NVIM_APPNAME=ranger nvim --clean -u "$$D/init.lua"'
tmux() {
  if [ $$# -eq 0 ]; then
    command tmux -L ranger -f "$$D/tmux.conf" \
      new-session -A -s main
  else
    command tmux -L ranger -f "$$D/tmux.conf" "$$@"
  fi
}
endef
export BASHRC

## vi ---------------------------------------------------------

F ?= $(firstword $(wildcard *.lua)) # for vi

vi: ## tuned nvim + catppuccin via vim.pack (wiped on exit)
	$(call need,nvim,vi)
	$(call need,git,vi)
	@D=$$(mktemp -d); trap "rm -rf $$D" EXIT; \
	 printf '%s\n' "$$INITLUA" > $$D/init.lua; \
	 NVIM_APPNAME=ranger nvim --clean -u $$D/init.lua $F

define INITLUA
vim.g.mapleader = " "
vim.g.maplocalleader = "\\"

vim.o.number = true
vim.o.relativenumber = true
vim.o.cursorline = true
vim.o.mouse = "a"
vim.o.termguicolors = true
vim.o.expandtab = true
vim.o.tabstop = 2
vim.o.shiftwidth = 2
vim.o.softtabstop = 2
vim.o.ignorecase = true
vim.o.smartcase = true
vim.o.scrolloff = 8
vim.o.signcolumn = "yes"
vim.o.splitbelow = true
vim.o.splitright = true
vim.o.wrap = true
vim.o.linebreak = true
vim.o.clipboard = "unnamedplus"
vim.o.fillchars = "vert:│,eob:·"
vim.o.winborder = "rounded"
vim.o.splitkeep = "screen"
vim.o.smoothscroll = true

vim.opt.packpath:prepend(vim.fn.stdpath("data") .. "/site")
vim.pack.add({
  "https://github.com/catppuccin/nvim",
  "https://github.com/nvim-lualine/lualine.nvim",
  "https://github.com/zaldih/themery.nvim",
  "https://github.com/folke/tokyonight.nvim",
  "https://github.com/rebelot/kanagawa.nvim",
  "https://github.com/rose-pine/neovim",
  "https://github.com/EdenEast/nightfox.nvim",
})
vim.cmd.colorscheme("catppuccin-mocha")
vim.cmd("hi! WinSeparator guifg=#7aa2f7 guibg=NONE")
vim.cmd("hi! VertSplit    guifg=#7aa2f7 guibg=NONE")
require("lualine").setup({ options = { theme = "auto" } })
require("themery").setup({
  themes = {
    "catppuccin-mocha","catppuccin-latte",
    "tokyonight-storm","tokyonight-day",
    "kanagawa-wave","kanagawa-dragon",
    "rose-pine","rose-pine-dawn",
    "nightfox","duskfox","carbonfox",
  },
  livePreview = true,
})

vim.api.nvim_create_autocmd("TextYankPost", {
  callback = function() vim.hl.on_yank() end,
})

vim.diagnostic.config({
  virtual_text     = false,
  signs            = true,
  underline        = true,
  update_in_insert = false,
  severity_sort    = true,
  float            = { border = "rounded" },
})

if vim.fn.executable("lua-language-server") == 1 then
  vim.lsp.config("luals", {
    cmd          = {"lua-language-server"},
    filetypes    = {"lua"},
    root_markers = {".luarc.json", ".git"},
    settings = { Lua = {
      diagnostics = { globals = {"vim"} },
      workspace   = { checkThirdParty = false },
      telemetry   = { enable = false },
    }},
  })
  vim.lsp.enable("luals")
  vim.api.nvim_create_autocmd("LspAttach", {
    callback = function(a)
      local c = vim.lsp.get_client_by_id(a.data.client_id)
      if c then c.server_capabilities.semanticTokensProvider = nil end
      local b = { buffer = a.buf, silent = true }
      vim.keymap.set("n", "K",         vim.lsp.buf.hover,         b)
      vim.keymap.set("n", "gd",        vim.lsp.buf.definition,    b)
      vim.keymap.set("n", "gr",        vim.lsp.buf.references,    b)
      vim.keymap.set("n", "<leader>e", vim.diagnostic.open_float, b)
    end,
  })
end
endef
export INITLUA

## tmux -------------------------------------------------------

tmux: ## tuned tmux on private socket (wiped on exit)
	$(call need,tmux,tmux)
	@D=$$(mktemp -d); trap "rm -rf $$D" EXIT; \
	 printf '%s\n' "$$TMUXCONF" > $$D/tmux.conf; \
	 tmux -L ranger -f $$D/tmux.conf new-session -A -s main

define TMUXCONF
# --- core ---
set -g  default-terminal "tmux-256color"
set -ag terminal-overrides ",xterm-256color:RGB,*:RGB"
set -g  mouse on
set -g  base-index 1
setw -g pane-base-index 1
set -g  renumber-windows on
set -g  escape-time 0
set -g  history-limit 50000
set -g  focus-events on
setw -g aggressive-resize on
setw -g mode-keys vi

# --- prefix: C-Space ---
unbind C-b
set -g prefix C-Space
bind C-Space send-prefix

# --- splits keep cwd; | vertical, - horizontal ---
unbind '"'
unbind %
bind '|' split-window -h -c "#{pane_current_path}"
bind '-' split-window -v -c "#{pane_current_path}"
bind '_' split-window -v -c "#{pane_current_path}"
bind  c  new-window      -c "#{pane_current_path}"

# --- sessions: fast make/jump ---
bind  C command-prompt -p "session:" "new-session -s '%%'"
bind  S choose-tree -Zs
bind -n M-Left  switch-client -p
bind -n M-Right switch-client -n

# --- reload (sources current active config file) ---
bind  r source-file "#{config_files}" \; display "reloaded #{config_files}"

# --- copy-mode vi ---
bind -T copy-mode-vi v send -X begin-selection
bind -T copy-mode-vi y send -X copy-selection-and-cancel

# --- catppuccin-mocha status bar ---
set -g  status-position top
set -g  status-justify  centre
set -g  status-style          "bg=#1e1e2e,fg=#cdd6f4"
set -g  window-status-style   "bg=#1e1e2e,fg=#a6adc8"
set -g  window-status-current-style "bg=#cba6f7,fg=#1e1e2e,bold"
set -g  window-status-format         " #I:#W "
set -g  window-status-current-format " #I:#W "
set -g  pane-border-style        "fg=#45475a"
set -g  pane-active-border-style "fg=#89b4fa"
set -g  message-style "bg=#cba6f7,fg=#1e1e2e,bold"
set -g  status-left-length  40
set -g  status-right-length 60
set -g  status-left  "#[bg=#89b4fa,fg=#1e1e2e,bold] #S #[bg=#1e1e2e,fg=#89b4fa]"
set -g  status-right "#[fg=#a6e3a1]#[bg=#a6e3a1,fg=#1e1e2e,bold] %H:%M #[bg=#a6e3a1,fg=#fab387]#[bg=#fab387,fg=#1e1e2e,bold] %d-%b "
endef
export TMUXCONF

## pdf --------------------------------------------------------

Font   ?= 5         # for ~/tmp/%.pdf
Cols   ?= 3         # for ~/tmp/%.pdf
Orient ?= landscape # for ~/tmp/%.pdf

~/tmp/%.pdf : %.lua Makefile ## lua -> pdf via a2ps
	$(call need,a2ps,pdf)
	$(call need,ps2pdf,pdf)
	@mkdir -p ~/tmp
	@echo "pdfing : $@ ..."
	@D=$$(mktemp -d); trap "rm -rf $$D" EXIT; \
	 mkdir -p $$D/.a2ps; \
	 echo "$$LUASSH" > $$D/.a2ps/lua.ssh; \
	 awk '/^-- @@/ {if (NR>1) printf "\f"; next} {print}' \
	     $< | \
	 HOME=$$D a2ps -Bj --$(Orient) --line-numbers=1 \
	      --highlight-level=normal --borders=no \
	      --pro=color --right-footer="" --left-footer="" \
	      --pretty-print=lua --footer="page %p." \
	      -M letter --font-size=$(Font) \
	      --columns $(Cols) -o - | ps2pdf - $@
	@$(OPEN) $@

define LUASSH
style Lua is
alphabets are
  "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_0"
case sensitive

keywords in Keyword are
  and,false,ipairs,nil,not,or,pairs,rawequal,rawget,rawlen,
  rawset,select,tonumber,tostring,true,type
end keywords

keywords in Keyword_strong are
  assert,break,collectgarbage,do,dofile,else,elseif,"end",
  error,for,function,getmetatable,goto,if,"in",local,load,
  loadfile,next,pcall,print,repeat,require,return,
  setmetatable,then,until,while,xpcall,_G,_VERSION
end keywords

keywords in Label_strong are "^" function end keywords
keywords in Comment       are self          end keywords

sequences are
  "[[" Comment "]]","--" Comment,C-string
end sequences
end style
endef
export LUASSH

ranger.lua

local the = {p=2, bins=5, least=nil}
local l = require"lib"

local push,csv,new,o,copy = l.push, l.csv, l.new, l.o, l.copy
local floor,sqrt,max,min = l.floor, l.sqrt, l.max, l.min
local abs,exp,log = l.abs, l.exp, l.log

-- ## Structs ---------------------------------------------------
local Sym,Num,Data,Cols = {},{},{},{}

function Sym.new(s,at)
  return new(Sym, {at=at or 0, txt=s or "", n=0, has={}, bins={}}) end

function Num.new(s,at,    bins)
  bins = {}
  for k=0, the.bins-1 do bins[k] = new(Num, {n=0, mu=0, m2=0}) end
  return new(Num,{at=at or 0, txt=s or "", n=0, mu=0, m2=0,
                  cuts={}, bins=bins,
                  goal=(s or ""):find"-$" and 0 or 1}) end

local function adds(src,it)
  it = it or Num.new()
  if type(src)=="function" then for x in src do it:add(x) end
  else for _,x in ipairs(src or {}) do it:add(x) end end
  return it end

function Data.new(src)
  return adds(src, new(Data, {cols=nil, _rows={}})) end

function Cols.new(row,     all,x,y,col)
  all,x,y = {},{},{}
  for at,s in ipairs(row) do
    col = push(all,(s:match"^[A-Z]" and Num or Sym).new(s,at))
    if not s:find"X$" then
      push(s:find"[-+!]$" and y or x,col) end end
  return new(Cols,{names=row, all=all, x=x, y=y}) end

function Data.clone(i,rows)
  return adds(rows or {}, Data.new{i.cols.names}) end

-- ## update ----------------------------------------------------
function Sym.add(i,v)
  if v=="?" or v==nil then return v end
  i.n,i.has[v] = i.n+1, 1+(i.has[v] or 0)
  return v end

function Num.add(i,v,    d)
  if v=="?" or v==nil then return v end
  i.n=i.n+1; d=v-i.mu; i.mu=i.mu+d/i.n; i.m2=i.m2+d*(v-i.mu)
  return v end

function Cols.add(i,row)
  for _,col in pairs(i.all) do col:add(row[col.at]) end end

-- accumulate sign into the bin for value v. Num bin pre-exists; Sym lazy.
function Num.addBin(i,v,sign) i.bins[v]:add(sign) end
function Sym.addBin(i,v,sign)
  i.bins[v] = i.bins[v] or new(Num,{n=0,mu=0,m2=0})
  i.bins[v]:add(sign) end

function Data.add(i,row)
  if not i.cols then i.cols = Cols.new(row) else
    i.cols:add(row)
    push(i._rows, row) end end

-- ## merge -----------------------------------------------------
-- pure: returns new Num. w=+1 add b into a; w=-1 remove b from a.
function Num.merge(a,b,w,    c,n,d)
  w = w or 1
  if b.n == 0 then return a end
  n = a.n + w*b.n
  if n <= 0 then return new(Num,{n=0,mu=0,m2=0}) end   -- subtraction empties a
  c   = copy(a)
  d   = b.mu - c.mu
  c.m2 = c.m2 + w*b.m2 + w*d*d*c.n*b.n/n
  c.mu = (c.mu*c.n + w*b.mu*b.n)/n
  c.n  = n
  return c end

-- ## methods ---------------------------------------------------
function Num.mid(i) return i.mu end
function Sym.mid(i,    most,out)
  most,out = -1,nil
  for k,n in pairs(i.has) do if n>most then out,most=k,n end end
  return out end

function Num.spread(i) return i.n<2 and 0 or (max(0,i.m2)/(i.n-1))^0.5 end
function Sym.spread(i,    e)
  e=0; for _,n in pairs(i.has) do e=e+ n/i.n * log(n/i.n,2) end
  return -e end

function Num.norm(i,v)
  if v=="?" then return v end
  v = (v - i.mu) / (i:spread() + 1E-32)
  return 1 / (1 + exp(-1.7 * max(-3, min(3, v)))) end

-- ## distance --------------------------------------------------
function Data.disty(i,row,  p,    d,n)
  d,n,p = 0,0,p or the.p
  for _,c in ipairs(i.cols.y) do
    n,d = n+1, d + abs(c:norm(row[c.at]) - c.goal)^p end
  return (d/n)^(1/p) end

function Data.sorty(i,rows,    y)
  y = function(row) return i:disty(row) end
  table.sort(rows, function(r1,r2) return y(r1) < y(r2) end)
  return rows end

-- bin: Sym -> value itself. Num -> 0..bins-1, also tracks real max in i.cuts[b].
function Sym.bin(_,v) return v end
function Num.bin(i,v,    b)
  if v=="?" then return v end
  b = floor(the.bins * i:norm(v))
  i.cuts[b] = max(i.cuts[b] or -math.huge, v)
  return b end

-- ## weigh -----------------------------------------------------
-- Walk all rows. Each col fills i.bins[v] = Num of y-distances (vs root stats).
-- Cut max variance-explained = standard CART regression split.
local function weigh(d, root,    v,y)
  root = root or d
  for _,row in pairs(d._rows) do
    y = root:disty(row)                       -- disty uses root's global col stats
    for _,col in pairs(d.cols.x) do
      v = col:bin(row[col.at])
      if v ~= "?" then col:addBin(v, y) end end end
  return d end

-- ## cuts ------------------------------------------------------
-- score = variance-explained = (lhs.mu^2 * lhs.n + rhs.mu^2 * rhs.n) / total.n
-- Same metric for Sym and Num. Bigger = cleaner separation of best vs rest.

-- helper: sum all bin Nums into one total
local function totalOf(bins,    total)
  total = new(Num,{n=0,mu=0,m2=0})
  for _,bin in pairs(bins) do total = total:merge(bin, 1) end
  return total end

-- if split passes min-size and beats best, return (sc, win); else pass through.
-- win = {label, lhs_mu, rhs_mu}. Caller picks yes-side = lower-mu side.
-- score = variance-explained = (lhs.mu^2 * lhs.n + rhs.mu^2 * rhs.n) / total.n
local function tryCut(lhs,rhs,total,least, label, best,bestv,    sc)
  if lhs.n < least or rhs.n < least then return best, bestv end
  sc = (lhs.mu*lhs.mu*lhs.n + rhs.mu*rhs.mu*rhs.n)/total.n
  if sc > best then return sc, {label=label, lhs_mu=lhs.mu, rhs_mu=rhs.mu} end
  return best, bestv end

-- pick yes-op = lower-mu side (since we minimize y).
local function pickSide(w, lo_op, hi_op)
  if w.lhs_mu <= w.rhs_mu then return lo_op, hi_op, w.lhs_mu end
  return hi_op, lo_op, w.rhs_mu end

function Sym.cut(i, least,    total,best,bestv,rhs,yes,no,mu)
  total = totalOf(i.bins)
  best, least = -math.huge, least or sqrt(total.n)
  for v,b in pairs(i.bins) do
    rhs = total:merge(b, -1)
    best,bestv = tryCut(b, rhs, total, least, v, best, bestv) end
  if not bestv then return nil end
  yes,no,mu = pickSide(bestv, "=", "~=")
  return {yes=yes, no=no, at=i.at, v=bestv.label, score=best, mu=mu} end

function Num.cut(i, least,    total,best,bestv,lhs,rhs,b,yes,no,mu)
  total = totalOf(i.bins)
  best, least = -math.huge, least or sqrt(total.n)
  lhs, rhs    = new(Num,{n=0,mu=0,m2=0}), total
  for k=0, the.bins-1 do
    b = i.bins[k]
    if b.n > 0 then
      lhs = lhs:merge(b,  1)
      rhs = rhs:merge(b, -1)
      best,bestv = tryCut(lhs, rhs, total, least, i.cuts[k], best, bestv) end end
  if not bestv then return nil end
  yes,no,mu = pickSide(bestv, "<=", ">")
  return {yes=yes, no=no, at=i.at, v=bestv.label, score=best, mu=mu} end

-- ## best cut over all x-cols ---------------------------------
local function bestCut(d,    out,cut)
  out = nil
  for _,col in pairs(d.cols.x) do
    cut = col:cut(the.least)
    if cut and (not out or cut.score > out.score) then out = cut end end
  return out end

-- ## scan: emit (low-mu cut, high-mu cut) per col, for FFT trees -------
-- Same bin walk as cut, but tracks lowest-mu AND highest-mu candidates.
local function trackExtremes(lhs,rhs,least, lo_op,hi_op, label, lo,hi,    L,H)
  if lhs.n < least or rhs.n < least then return lo, hi end
  if lhs.mu <= rhs.mu then L,H = lhs.mu, rhs.mu
  else                     L,H = rhs.mu, lhs.mu
                           lo_op,hi_op = hi_op,lo_op end
  if not lo or L < lo.mu then lo = {yes=lo_op,no=hi_op,v=label,mu=L} end
  if not hi or H > hi.mu then hi = {yes=hi_op,no=lo_op,v=label,mu=H} end
  return lo, hi end

function Sym.scan(i, least,    total,lo,hi,rhs)
  total = totalOf(i.bins)
  least = least or sqrt(total.n)
  for v,b in pairs(i.bins) do
    rhs = total:merge(b, -1)
    lo,hi = trackExtremes(b, rhs, least, "=","~=", v, lo, hi) end
  if lo then lo.at = i.at end
  if hi then hi.at = i.at end
  return lo, hi end

function Num.scan(i, least,    total,lo,hi,lhs,rhs,b)
  total = totalOf(i.bins)
  least = least or sqrt(total.n)
  lhs, rhs = new(Num,{n=0,mu=0,m2=0}), total
  for k=0, the.bins-1 do
    b = i.bins[k]
    if b.n > 0 then
      lhs = lhs:merge(b,  1)
      rhs = rhs:merge(b, -1)
      lo,hi = trackExtremes(lhs, rhs, least, "<=",">", i.cuts[k], lo, hi) end end
  if lo then lo.at = i.at end
  if hi then hi.at = i.at end
  return lo, hi end

-- global best (lowest mu yes-side) and worst (highest mu yes-side) cut
local function extremes(d,    best,worst,lo,hi)
  for _,col in pairs(d.cols.x) do
    lo, hi = col:scan(the.least)
    if lo and (not best  or lo.mu < best.mu ) then best  = lo end
    if hi and (not worst or hi.mu > worst.mu) then worst = hi end end
  return best, worst end

-- ## show ------------------------------------------------------
local function show(cut, d)
  return d.cols.names[cut.at].." "..cut.yes.." "..o(cut.v) end

-- ## decision list --------------------------------------------
-- does row satisfy cut.yes side?
local function matches(row, cut,    v)
  v = row[cut.at]
  if v == "?"        then return false end
  if cut.yes == "="  then return v == cut.v end
  if cut.yes == "~=" then return v ~= cut.v end
  if cut.yes == "<=" then return v <= cut.v end
  if cut.yes == ">"  then return v >  cut.v end end

-- mean of each y-col on the given rows
local function yMeans(d, rows,    out,col)
  out = {}
  for _,c in ipairs(d.cols.y) do
    col = new(Num,{n=0,mu=0,m2=0})
    for _,row in pairs(rows) do
      if row[c.at] ~= "?" then col:add(row[c.at]) end end
    out[1+#out] = c.txt.."="..o(col.mu) end
  return table.concat(out, " ") end

-- Num summarizing disty of given rows (vs root's global stats)
local function dyStat(root, rows,    s)
  s = new(Num,{n=0,mu=0,m2=0})
  for _,row in pairs(rows) do s:add(root:disty(row)) end
  return s end

-- refinement chain: each rule narrows the YES-side of the previous best cut.
-- conjunctive: line i+1 sits inside subset picked by lines 0..i.
local function decisionList(d, depth, sd0, root,    cut,yes,no,parent,rule,muYes)
  root   = root or d
  parent = dyStat(root, d._rows)
  if not sd0 then
    sd0 = parent:spread()
    print(string.format("%-5s%-44s | n=%-7d d2h=%.2f sd=%.2f  %s",
          "BASE", "", #d._rows, parent.mu, sd0, yMeans(d, d._rows))) end
  cut = bestCut(weigh(d, root))
  if not cut then return end
  yes, no = {}, {}
  for _,row in pairs(d._rows) do
    push(matches(row, cut) and yes or no, row) end
  muYes = dyStat(root, yes).mu
  rule  = depth==0 and "IF" or "AND"
  print(string.format("%-5s%-44s | n=%-7d d2h=%.2f           %s",
        rule, show(cut, d), #yes, muYes, yMeans(d, yes)))
  if #yes >= 2*(the.least or sqrt(#d._rows)) then
    decisionList(Data.clone(d, yes), depth+1, sd0, root) end end

-- ## FFT tree ---------------------------------------------------
-- Each internal node: {cut, leaf=Num-of-disty(yes), sub=child-tree, bias}
-- Each terminal: {leaf=Num-of-disty(rows)}
-- depth d → enumerates up to 2^d trees. Skeleton; heavy lifting in weigh+scan.
local function fftBuild(d, depth, root,    out,best,worst,yes,no,leafNum,subs)
  root = root or d
  if depth <= 0 or #d._rows < 2*(the.least or sqrt(#d._rows)) then
    return {{leaf=dyStat(root, d._rows), n=#d._rows}} end
  best, worst = extremes(weigh(d, root))
  out = {}
  for _, pair in ipairs({{best,"best"},{worst,"worst"}}) do
    local choice, bias = pair[1], pair[2]
    if choice then
      yes, no = {}, {}
      for _,row in pairs(d._rows) do
        push(matches(row, choice) and yes or no, row) end
      if #yes > 0 and #no >= 2 then
        leafNum = dyStat(root, yes)
        subs = fftBuild(Data.clone(d, no), depth-1, root)
        for _,sub in ipairs(subs) do
          push(out, {cut=choice, leaf=leafNum, n=#yes, sub=sub, bias=bias}) end end end end
  if #out == 0 then return {{leaf=dyStat(root, d._rows), n=#d._rows}} end
  return out end

-- predict: walk tree, return leaf.mu at first matching cut, else descend.
local function fftPredict(tree, row)
  while tree.cut do
    if matches(row, tree.cut) then return tree.leaf.mu end
    tree = tree.sub end
  return tree.leaf.mu end

-- MAE of tree predictions vs actual disty across rows.
local function fftScore(tree, root,    err)
  err = 0
  for _,row in pairs(root._rows) do
    err = err + abs(fftPredict(tree, row) - root:disty(row)) end
  return err / #root._rows end

local function fftBest(trees, root,    win,winSc,sc)
  winSc = math.huge
  for _,t in ipairs(trees) do
    sc = fftScore(t, root)
    if sc < winSc then winSc, win = sc, t end end
  return win, winSc end

local function fftShow(tree, d, depth)
  depth = depth or 0
  if tree.cut then
    print(string.format("%s[%-5s] IF %-30s THEN d2h=%.3f (n=%d) ELSE",
          ("  "):rep(depth), tree.bias, show(tree.cut, d), tree.leaf.mu, tree.n))
    fftShow(tree.sub, d, depth+1)
  else
    print(string.format("%sLEAF d2h=%.3f (n=%d)",
          ("  "):rep(depth), tree.leaf.mu, tree.n)) end end

-- ## Start-up -------------------------------------------------
local t0 = os.clock()
local d  = Data.new(csv("~/gits/moot/optimize/misc/auto93.csv"))
print("=== decision list (refinement chain) ===")
decisionList(d, 0)
print()
print("=== FFT tree (best of 2^d candidates) ===")
local trees = fftBuild(d, 4)
local best, sc = fftBest(trees, d)
print(string.format("trees evaluated: %d   best MAE: %.4f", #trees, sc))
fftShow(best, d)
print(string.format("runtime=%.3fs", os.clock()-t0))

writ.md

Style + Contributing -- read fast, write compact

Cheat sheet for parsing AND writing code in this gist.

Smoosh rule: this doc itself follows the code style -- terse, dense, no filler. When extending it, drop articles/redundancy, favor fragments + tables, max info/line.

Philosophy

• COI compose, not inherit
• LoB (Locality of Behavior) a type's methods + the funcs operating on it live together (wrapper next to its algorithm). Exception: cross-type families group by op -- all .new ctors, all .add/adds may cluster.
• SSOT config from one help str
• BOB funcs <=5 lines -- warn not mandatory
• BAIL one-line guards
• KISS one func, one job
• R3 rule of three: don't wrap until 3rd use; 1-2 = inline (egs don't count as "real" uses)
• DSL when a pattern repeats, invent tiny notation + write ONE interpreter. Keep the notation as data (string/table); the engine stays small. See ## DSLs below.
• ZIP max logic/screen, min whitespace
• HINT names = types
• ASCII only (no ->, x etc -- break a2ps PDF)
• Indent 2, width <=65 everywhere; local all top funcs

DSLs (notation + one parser)

Recurring shapes encoded as data, each read by one small engine:

notation	example	parser
CSV header = schema	Clndrs Lbs- Mpg+ klass! (Caps=Num, X/-/+/!=skip/min/max/klass)	Cols.new
per-file help string = help+config	local help=[[\n## options\n --bins=10 desc\n## egs\n --foo demo\n]]	l.section/l.flags/l.help
test cases = triples	{"mid", sym:mid(), "a"}	l.chk
Makefile self-doc	target: ## desc, Var ?= v # for X	help gawk

Spot the repetition -> minimal syntax -> single interpreter. Adding a row/col/test/flag = new data, no new code.

Names (HINT)

• i = self (every method; ctors don't take self)
• n = int/count/idx | s/v = str/scalar
• t = generic table | fn/ok = cb/bool
• at = col idx | lo/hi = bounds | mu/sd = stats
• num/sym/col/node/row = domain instances
• Plurals (cols,rows,xs=feats,ys=goals) = lists

Casing

• Pascal = class + ctor (Sym,Num,Confuse)
• camel = instance | m = own module exports
• l = require"lib" | luamine = require"luamine"

Function signatures

• One-line -- comment above (no blank between)
• Locals declared in arg list after 4-space GAP
• Return type via --> ...

Short:

-- entropy in bits
function ent(t,    e,N) --> qty

Long arg/local list: wrap; return type on the sig's last line:

-- best (col,v) minimizing weighted y-spread over a split
function m.cutsBest(cols,rows,y,Sumr,
                    best,score,bins,tot,s) --> best|nil

Type vocab: tbl/qty/bool/str/v/fn/nil + domain types. Unions: a|b.

Class-as-API (Sym/Num/Confuse pattern)

local Sym = {}
m.Sym = Sym
function Sym.new(s,at)   return l.new(Sym, {...}) end
function Sym.add(i,v)    i.has[v] = ... end   -- :add
function Sym.mid(i)      return ... end       -- :mid

• ctor .new returns instance via l.new(mt, t)
• methods take i as first arg -> callable as sym:add(v)
• group by lifecycle: ctor -> update -> query
• exports grouped before l.boot (m.Sym,m.Num,... = Sym,Num,...) so they exist when egs run

File skeleton

#!/usr/bin/env lua
-- conventions:
--   tiny.cc/ranger#file-claude-md
local help = [[
# one-line title
(c) 2026 Tim Menzies <timm@ieee.org> MIT license
## options
  -t  --train=...  train CSV
      --seed=1     RNG seed
      --foo=42     this file's knob
## egs
  --foo   foo demo
]]
local b4={}; for k,_ in pairs(_G) do b4[k]=k end
local m = {}
local floor = math.floor             -- aliases
local l    = require"lib"            -- skip in lib.lua
local the  = l.the                   -- shared config

-- ## section ----
function m.foo(...) ... end

-- ## eg ----
local eg = {}
eg["--foo"] = function() ... return l.chk(...) end

l.boot(eg,b4,"FILE",help)            -- tail = 2 lines
return m

Order: help string -> b4 snapshot -> m -> aliases -> require -> the=l.the -> funcs -> egs -> l.boot -> return. (lib.lua differs: it owns local m,the={},{} and sets m.the=the.)

help string is SSOT for help + defaults. l.boot parses its ## options section and seeds the. Each file declares every knob its CLI accepts -- including shared (--seed, -t, -T) which appear redundantly in multiple files; keep them in sync (mechanical or LLM-driven).

Domain funcs take config as args where it matters (multi-level or per-call override: y/leaf/Sumr). Deep single-consumer defaults may read the at the leaf (bins/p: x or the.x).

CLI / help

• Per-file local help = [[...]] string is the SSOT. Sections: # title (one line), (c) ... (one line), ## options (flag decls), ## egs (eg docs). l.boot(eg,b4,name,help) parses it via m.section/m.flags.
• One shared config: the lives in lib (l.the); each file does local the = l.the. Each file's l.boot merges its own ## options defaults into the (last loader wins on overlap).
• Option line: [ -x ] --longkey[=default] desc. Lines with =default seed the; action lines (no =) don't.
• Flags are long-only (--bins) except canonical shorts -t(train), -T(test), -h(help). No first-char derivation; unknown flag = assert "bad flag". No single-letter collisions.
• Action: --<name> runs eg["--<name>"]()
• l.main auto-intercepts: --all, --the, -h/--help. Don't define these as egs.
• l.help(name,h) prints title (# ...), (c) line, OPTIONS (## options), ACTIONS (## egs) -- all from the caller's header h.
• make check-headers lints per-file: eg["--X"] defined but not in ## egs = FAIL; ## egs --X with no eg["--X"] = warn.

Eg pattern (chk + pcall driver)

Each eg returns (flag,msg) via l.chk. Driver pcalls + tracks.

eg["--Sym"] = function(    sym)
  sym = Sym.new()
  for _,v in ipairs{"a","a","b","a","c"} do sym:add(v) end
  return l.chk({"mid",      sym:mid(),      "a"},
               {"spread>0", sym:spread()>0, true},
               {"bin id",   sym:bin("z"),   "z"}) end

• Each case: {tag, got, want} triple
• l.chk prints each tag = got then asserts got==want
• First-fail returns (false, tag); pass returns true
• Print-only egs (m.show(root, cols)): end with return true
• Single-eg run: silent on pass, "FAIL ..." on fail
• --all: prints -- name per eg, summary + fail list

Module convention

• lib.lua -- generic helpers (list/stats/csv/cli/o/Confuse). Self-contained.
• luamine.lua -- AI primitives (Num/Sym/Cols/dist*/tree/...). require"lib".
• Apps (ltree.lua, ...) -- one task per file. require"lib" + "luamine".

Each file standalone-runnable AND importable.

CSV

Header row names cols. Caps first letter = Num, else Sym. Suffix: X=skip, -=goal min, +=goal max, !=klass. Rows sorted ascending by disty (lower = better).

Clndrs, Volume, HpX, Lbs-, Acc+, Mpg+
4,      90,     48,  1985, 21.5, 40

Header file (gist landing)

Filename starts with , (comma, ASCII 44) so it sorts first. Avoid + (vim cmd flag), ! (shell history), ~ (ambiguous). Contains: copyright HTML, hero image, badge row, recursive subtitle, short URL, Files TOC.

TOC anchors: gist format #file-<name>-<ext> (lowercase, dots/punct stripped).

Section markers

Two levels:

• -- ## name ---- subsection (groups related funcs; LoB)
• -- @@ name ---- column break for make pdf. Place at the start of each PDF column-group (multiple -- ## may share one column). Awk in pdf target inserts \f here; line itself stripped.

Makefile

• Modeline # vim: ts=2 sw=2 sts=2 et : + SHELL := /bin/bash
• Self-doc: target: ## desc parsed by help (gawk)
• Defaults: Var ?= val # for <rule> -- help shows them in auto-generated defaults section. SSOT.
• Silent recipes via @ prefix; $$ for shell vars
• Tool guard: $(call need,tool,target)
• Multi-line shell blobs: define X ... endef + export X
• No temp files in ~. Use mktemp -d+trap "rm -rf $$D" EXIT
• Pattern rules: ~/tmp/%.pdf : %.lua Makefile
• Static-pattern (shared body, multi-target): cv-c cv-r: cv-%: $(DATA)
• ANSI color: \033[36m...\033[0m

luacheck

Ignores live in Makefile check target (var Ignore). No .luacheckrc, no per-file -- luacheck: ignore. SSOT.

On "update"

1. Re-read this file
2. Audit every file in gist
3. Print plan of changes (file -> what)
4. Pause for confirmation
5. On approval: apply

Known fossils

• Per-func -- comments missing in older code
• --> return annotations not retro-applied everywhere