Skip to content

Instantly share code, notes, and snippets.

@ShanonJackson

ShanonJackson/lexer.c

Last active July 22, 2026 23:40
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save ShanonJackson/bf90cb316c4fab1b8772b4dc8aeb326a to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save ShanonJackson/bf90cb316c4fab1b8772b4dc8aeb326a to your computer and use it in GitHub Desktop.
Download ZIP
x86_64 AMD ZEN 3 Lexing JS at 5-7GB/S (Just boundary finding + tokenization) - Purposely excludes some things for demonstration purposes.
Raw
lexer.c
/*
* SIMD JavaScript/TypeScript lexer for x86-64 AVX2, tuned for AMD Zen 3 <<< IMPORTANT
*
* CccLex L = {0};
* uint8_t *buf = ccc_source_alloc(n); // 64-byte aligned, 64B zero-padded
* memcpy(buf, bytes, n);
* size_t ntok = ccc_lex(&L, buf, n); // tokens: L.starts[i], L.kinds[i]
* ccc_source_free(buf);
* ccc_free(&L);
*
* src must be 64-byte aligned with 64 zero bytes after src[n-1]; ccc_source_alloc
* and ccc_read_file return a suitable buffer. starts[i] is a byte offset, kinds[i]
* is the enum below. Each whitespace run is one WS token.
*
* cc -O3 -march=znver3 lexer.c -o ccc
*/
#define _POSIX_C_SOURCE 199309L
#include <immintrin.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <x86intrin.h>
enum { WS, IDENT, NUM, STR, TMPL, LCOM, BCOM, REGEX, PUNCT };
static inline void *ccc_alloc64(size_t size) {
#ifdef _WIN32
return _aligned_malloc(size, 64);
#else
return aligned_alloc(64, (size + 63) & ~(size_t)63);
#endif
}
static inline void ccc_afree(void *p) {
#ifdef _WIN32
_aligned_free(p);
#else
free(p);
#endif
}
uint8_t *ccc_source_alloc(size_t n) {
uint8_t *b = (uint8_t *)ccc_alloc64(n + 64);
if (b) memset(b + n, 0, 64);
return b;
}
void ccc_source_free(uint8_t *b) { ccc_afree(b); }
static inline bool is_regex_keyword(const uint8_t *p, size_t len) {
switch (len) {
case 2: return !memcmp(p, "in", 2) || !memcmp(p, "of", 2) || !memcmp(p, "do", 2);
case 3: return !memcmp(p, "new", 3);
case 4: return !memcmp(p, "case", 4) || !memcmp(p, "void", 4) || !memcmp(p, "else", 4);
case 5: return !memcmp(p, "yield", 5) || !memcmp(p, "await", 5) || !memcmp(p, "throw", 5);
case 6: return !memcmp(p, "return", 6) || !memcmp(p, "typeof", 6) || !memcmp(p, "delete", 6);
case 10: return !memcmp(p, "instanceof", 10);
default: return false;
}
}
static inline uint64_t bm_lo_mask(size_t b) {
return ~((~(uint64_t)1) << (b & 63));
}
static inline int64_t bm_prev1(const uint64_t *bm, size_t p) {
if (p == 0) return -1;
size_t i = p - 1;
int64_t w = (int64_t)(i >> 6);
uint64_t x = bm[w] & bm_lo_mask(i);
for (; w >= 0; w--, x = bm[w])
if (x) return (w << 6) + (63 - (int64_t)__builtin_clzll(x));
return -1;
}
static inline void bm_set1(uint64_t *bm, size_t i) { bm[i >> 6] |= (uint64_t)1 << (i & 63); }
static inline size_t bm_next0(const uint64_t *bm, size_t i, size_t n) {
size_t w = i >> 6;
uint64_t inv = ~bm[w] & ~((((uint64_t)1 << (i & 63)) - 1));
while (!inv) {
if ((++w << 6) >= n) return n;
inv = ~bm[w];
}
size_t r = (w << 6) + (size_t)__builtin_ctzll(inv);
return r < n ? r : n;
}
static inline void bm_clear_range(uint64_t *bm, size_t a, size_t b) {
if (a > b) return;
size_t wa = a >> 6, wb = b >> 6;
uint64_t lo = ~(uint64_t)0 << (a & 63);
uint64_t hi = bm_lo_mask(b);
if (wa == wb) { bm[wa] &= ~(lo & hi); return; }
bm[wa] &= ~lo;
volatile uint64_t *vm = bm;
for (size_t w = wa + 1; w < wb; w++) vm[w] = 0;
bm[wb] &= ~hi;
}
static inline __m256i veq(__m256i v, uint8_t c) {
return _mm256_cmpeq_epi8(v, _mm256_set1_epi8((char)c));
}
static inline __m256i vwordbits(__m256i v) {
const __m256i lo_tbl = _mm256_setr_epi8(
(char)0xa4, (char)0xa6, (char)0xa6, (char)0xa6, (char)0xb6, (char)0xa6,
(char)0xa6, (char)0xa6, (char)0xa6, (char)0xa6, 0x26, 0x22, 0x22, 0x22, 0x22, 0x2a,
(char)0xa4, (char)0xa6, (char)0xa6, (char)0xa6, (char)0xb6, (char)0xa6,
(char)0xa6, (char)0xa6, (char)0xa6, (char)0xa6, 0x26, 0x22, 0x22, 0x22, 0x22, 0x2a);
const __m256i hi_tbl = _mm256_setr_epi8(
0x00, 0x00, 0x10, (char)0x80, 0x02, 0x0c, 0x02, 0x04,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
0x00, 0x00, 0x10, (char)0x80, 0x02, 0x0c, 0x02, 0x04,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20);
const __m256i nib = _mm256_set1_epi8(0x0f);
__m256i lo = _mm256_shuffle_epi8(lo_tbl, _mm256_and_si256(v, nib));
__m256i hi = _mm256_shuffle_epi8(hi_tbl,
_mm256_and_si256(_mm256_srli_epi16(v, 4), nib));
return _mm256_and_si256(lo, hi);
}
static inline __m256i vws(__m256i v) {
const __m256i t = _mm256_setr_epi8(
0x20, -128, -128, -128, -128, -128, -128, -128,
-128, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, -128, -128,
0x20, -128, -128, -128, -128, -128, -128, -128,
-128, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, -128, -128);
return _mm256_cmpeq_epi8(_mm256_shuffle_epi8(t, v), v);
}
static inline __m256i vopener(__m256i v) {
const __m256i t = _mm256_setr_epi8(
0x60, -128, 0x22, -128, -128, -128, -128, 0x27,
-128, -128, -128, -128, -128, -128, -128, 0x2f,
0x60, -128, 0x22, -128, -128, -128, -128, 0x27,
-128, -128, -128, -128, -128, -128, -128, 0x2f);
return _mm256_cmpeq_epi8(_mm256_shuffle_epi8(t, v), v);
}
static inline uint32_t mm(__m256i v) { return (uint32_t)_mm256_movemask_epi8(v); }
static inline __m256i load256(const uint8_t *src, size_t i) {
return _mm256_loadu_si256((const __m256i *)(src + i));
}
static inline __m256i bcast_at(const uint8_t *p) {
#if defined(__GNUC__)
__m256i v;
__asm__("vpbroadcastb %1, %0" : "=x"(v) : "m"(*p));
return v;
#else
return _mm256_set1_epi8((char)*p);
#endif
}
static inline size_t find_any(const uint8_t *src, size_t n, size_t i,
uint8_t a, uint8_t b, uint8_t c, uint8_t d) {
while (i + 32 <= n) {
__m256i v = load256(src, i);
uint32_t m = mm(_mm256_or_si256(_mm256_or_si256(veq(v, a), veq(v, b)),
_mm256_or_si256(veq(v, c), veq(v, d))));
if (m) return i + (size_t)__builtin_ctz(m);
i += 32;
}
for (; i < n; i++) {
uint8_t x = src[i];
if (x == a || x == b || x == c || x == d) return i;
}
return n;
}
static size_t scan_quoted(const uint8_t *src, size_t n, size_t i, uint8_t q) {
__m256i vq = bcast_at(&src[i - 1]);
const __m256i vb = _mm256_set1_epi8('\\');
(void)q;
for (;;) {
while (i + 32 <= n) {
__m256i v = load256(src, i);
uint32_t mq = mm(_mm256_cmpeq_epi8(v, vq));
uint32_t mb = mm(_mm256_cmpeq_epi8(v, vb));
uint32_t m = mq | mb;
if (m) {
uint32_t t = (uint32_t)__builtin_ctz(m);
if ((mb >> t) & 1) { i += t + 2; goto rescan; }
return i + t;
}
i += 32;
}
for (; i < n; i++) {
uint8_t x = src[i];
if (x == '\\') { i++; continue; }
if (x == q) return i;
}
return n;
rescan:;
}
}
static size_t scan_block(const uint8_t *src, size_t n, size_t i) {
for (;;) {
size_t p = find_any(src, n, i, '/', '/', '/', '/');
if (p >= n) return n;
if (p > i && src[p - 1] == '*') return p;
i = p + 1;
}
}
static inline size_t scan_block_ops(const uint8_t *src, size_t n,
const uint64_t *op, size_t nb,
size_t w, uint64_t bits, size_t s) {
uint64_t b = bits & (bits - 1);
int miss = 0;
for (;;) {
while (b == 0) {
if (++w >= nb) return n;
b = op[w];
}
size_t p = (w << 6) + (size_t)__builtin_ctzll(b);
uint16_t pair;
memcpy(&pair, src + p - 1, 2);
if (p > s + 2 && pair == 0x2F2A) return p;
if (++miss == 3) return scan_block(src, n, p + 1);
b &= b - 1;
}
}
static inline size_t find_any_re(const uint8_t *src, size_t n, size_t i) {
const __m256i t = _mm256_setr_epi8(
-128, -128, -128, -128, -128, -128, -128, -128,
-128, -128, -128, 0x5b, 0x5c, 0x5d, -128, 0x2f,
-128, -128, -128, -128, -128, -128, -128, -128,
-128, -128, -128, 0x5b, 0x5c, 0x5d, -128, 0x2f);
while (i + 32 <= n) {
__m256i v = load256(src, i);
uint32_t m = mm(_mm256_cmpeq_epi8(_mm256_shuffle_epi8(t, v), v));
if (m) return i + (size_t)__builtin_ctz(m);
i += 32;
}
for (; i < n; i++) {
uint8_t x = src[i];
if (x == '/' || x == '\\' || x == '[' || x == ']') return i;
}
return n;
}
static size_t scan_regex(const uint8_t *src, size_t n, size_t i) {
bool incl = false;
for (;;) {
size_t p = find_any_re(src, n, i);
if (p >= n) return n;
uint8_t c = src[p];
if (c == '\\') { i = p + 2; continue; }
if (incl) { if (c == ']') incl = false; i = p + 1; continue; }
if (c == '[') { incl = true; i = p + 1; continue; }
if (c == '/') return p;
i = p + 1;
}
}
static inline void classify_block(const uint8_t *src, size_t i, size_t blk,
uint64_t *word, uint64_t *st, uint64_t *op,
uint8_t *kind, uint64_t *pcw, uint64_t *pcs) {
const __m256i v_ws = _mm256_set1_epi8(WS);
const __m256i v_ident = _mm256_set1_epi8(IDENT);
const __m256i v_num = _mm256_set1_epi8(NUM);
const __m256i v_punct = _mm256_set1_epi8(PUNCT);
__m256i v0 = load256(src, i);
__m256i v1 = load256(src, i + 32);
__m256i b0 = vwordbits(v0), b1 = vwordbits(v1);
const __m256i k7f = _mm256_set1_epi8(0x7f);
__m256i w0 = _mm256_add_epi8(b0, k7f), w1 = _mm256_add_epi8(b1, k7f);
__m256i s0 = vws(v0), s1 = vws(v1);
__m256i d0 = b0;
__m256i d1 = b1;
__m256i q0 = vopener(v0), q1 = vopener(v1);
uint64_t wordm = (uint64_t)mm(w0) | ((uint64_t)mm(w1) << 32);
uint64_t wsm = (uint64_t)mm(s0) | ((uint64_t)mm(s1) << 32);
word[blk] = wordm;
op[blk] = (uint64_t)mm(q0) | ((uint64_t)mm(q1) << 32);
uint64_t wprev = (wordm << 1) | *pcw;
uint64_t sprev = (wsm << 1) | *pcs;
*pcw = wordm >> 63;
*pcs = wsm >> 63;
st[blk] = ~((wsm & sprev) | (wordm & wprev));
__m256i k0 = v_punct;
k0 = _mm256_blendv_epi8(k0, v_ws, s0);
k0 = _mm256_blendv_epi8(k0, v_ident, w0);
k0 = _mm256_blendv_epi8(k0, v_num, d0);
__m256i k1 = v_punct;
k1 = _mm256_blendv_epi8(k1, v_ws, s1);
k1 = _mm256_blendv_epi8(k1, v_ident, w1);
k1 = _mm256_blendv_epi8(k1, v_num, d1);
_mm256_storeu_si256((__m256i *)(kind + blk * 64), k0);
_mm256_storeu_si256((__m256i *)(kind + blk * 64 + 32), k1);
}
static void classify(const uint8_t *src, size_t n, uint64_t *word, uint64_t *st,
uint64_t *op, uint8_t *kind) {
uint64_t cw = 0, cs = 0;
size_t i = 0, blk = 0;
for (; i + 64 <= n; i += 64, blk++)
classify_block(src, i, blk, word, st, op, kind, &cw, &cs);
if (i < n) {
classify_block(src, i, blk, word, st, op, kind, &cw, &cs);
uint64_t mask = ((uint64_t)1 << (n - i)) - 1;
word[blk] &= mask;
op[blk] &= mask;
st[blk] &= mask;
}
}
static bool prev_is_regex(const uint8_t *src, const uint64_t *st, const uint8_t *kind,
const uint64_t *word, size_t n, size_t p) {
for (int64_t q = bm_prev1(st, p); q >= 0; q = bm_prev1(st, (size_t)q)) {
size_t qi = (size_t)q;
uint8_t k = kind[qi];
if (k == WS || k == LCOM || k == BCOM) continue;
if (k == STR || k == REGEX || k == TMPL || k == NUM) return false;
if (k == IDENT) return is_regex_keyword(src + qi, bm_next0(word, qi, n) - qi);
if (k == PUNCT) return !(src[qi] == ')' || src[qi] == ']');
return true;
}
return true;
}
static void carve(const uint8_t *src, size_t n, uint64_t *st, uint8_t *kind,
const uint64_t *word, const uint64_t *op) {
size_t nb = (n + 63) >> 6;
for (size_t w = 0; w < nb; w++) {
uint64_t bits = op[w];
while (bits) {
size_t s = (w << 6) + (size_t)__builtin_ctzll(bits);
uint8_t c = src[s], kd;
size_t end;
if (c == '/') {
uint8_t d = src[s + 1];
if (d == '/') {
end = find_any(src, n, s + 2, '\n', '\r', '\n', '\r');
kind[s] = LCOM;
if (end > s + 1) bm_clear_range(st, s + 1, end - 1);
if (end < n) bm_set1(st, end);
goto advance;
}
if (d == '*') {
size_t e = scan_block_ops(src, n, op, nb, w, bits, s);
end = (e < n) ? e + 1 : n;
kd = BCOM;
} else if (prev_is_regex(src, st, kind, word, n, s)) {
size_t e = scan_regex(src, n, s + 1);
end = (e < n) ? e + 1 : n;
if (end < n && ((word[end >> 6] >> (end & 63)) & 1))
end = bm_next0(word, end, n);
kd = REGEX;
} else {
bits &= bits - 1;
continue;
}
} else {
size_t e = scan_quoted(src, n, s + 1, c);
end = (e < n) ? e + 1 : n;
kd = (c == '`') ? TMPL : STR;
}
kind[s] = kd;
if (end > s + 1) bm_clear_range(st, s + 1, end - 1);
advance:
if (end >= n) return;
size_t we = end >> 6;
if (we == w) {
bits &= ~(uint64_t)0 << (end & 63);
} else {
w = we;
bits = op[w] & (~(uint64_t)0 << (end & 63));
}
}
}
}
static _Alignas(64) uint8_t ccc_lut0z[256][8];
static _Alignas(64) uint8_t ccc_lutpad[256][32];
static void build_pair_luts(void) {
for (size_t m = 0; m < 256; m++) {
size_t k = 0;
memset(ccc_lut0z[m], 0, 8);
memset(ccc_lutpad[m], 0, 32);
for (size_t bit = 0; bit < 8; bit++)
if ((m >> bit) & 1) {
ccc_lut0z[m][k] = (uint8_t)bit;
ccc_lutpad[m][8 + k] = (uint8_t)(bit + 8);
k++;
}
for (size_t j = k; j < 8; j++) ccc_lutpad[m][8 + j] = 0x80;
}
}
static inline __m128i compose_ctrl(unsigned sub0, unsigned sub1, unsigned pc0) {
__m128i row0 = _mm_loadl_epi64((const __m128i *)ccc_lut0z[sub0]);
__m128i row1 = _mm_loadu_si128((const __m128i *)(ccc_lutpad[sub1] + 8 - pc0));
return _mm_or_si128(row0, row1);
}
static size_t compress(const uint64_t *st, const uint8_t *kind, size_t nb,
uint32_t *starts, uint8_t *kinds) {
const __m256i k16 = _mm256_set1_epi32(16);
size_t m = 0;
for (size_t b = 0; b < nb; b++) {
uint64_t mword = st[b];
if (mword == 0) continue;
__m256i pb = _mm256_set1_epi32((int)(b * 64));
__m128i ctrl[4];
size_t cur[4];
for (size_t p = 0; p < 4; p++) {
unsigned sub0 = (unsigned)((mword >> (16 * p)) & 0xff);
unsigned sub1 = (unsigned)((mword >> (16 * p + 8)) & 0xff);
unsigned pc0 = (unsigned)__builtin_popcount(sub0);
__m128i c = compose_ctrl(sub0, sub1, pc0);
ctrl[p] = c;
cur[p] = m;
_mm256_storeu_si256((__m256i *)(starts + m),
_mm256_add_epi32(pb, _mm256_cvtepu8_epi32(c)));
_mm256_storeu_si256((__m256i *)(starts + m + 8),
_mm256_add_epi32(pb, _mm256_cvtepu8_epi32(_mm_srli_si128(c, 8))));
m += pc0 + (unsigned)__builtin_popcount(sub1);
pb = _mm256_add_epi32(pb, k16);
}
for (size_t p = 0; p < 4; p++) {
__m128i kb = _mm_loadu_si128((const __m128i *)(kind + b * 64 + 16 * p));
_mm_storeu_si128((__m128i *)(kinds + cur[p]), _mm_shuffle_epi8(kb, ctrl[p]));
}
}
return m;
}
typedef struct {
uint32_t *starts;
uint8_t *kinds;
uint64_t *word, *st, *op;
uint8_t *kind;
size_t nb_cap, out_cap;
} CccLex;
void ccc_free(CccLex *L) {
free(L->word); free(L->st); free(L->op); free(L->kind);
free(L->starts); free(L->kinds);
memset(L, 0, sizeof(*L));
}
static void ccc_ensure(CccLex *L, size_t n) {
size_t nb = (n + 63) / 64 + 1;
if (L->nb_cap < nb) {
L->word = (uint64_t *)realloc(L->word, nb * sizeof(uint64_t));
L->st = (uint64_t *)realloc(L->st, nb * sizeof(uint64_t));
L->op = (uint64_t *)realloc(L->op, nb * sizeof(uint64_t));
L->kind = (uint8_t *)realloc(L->kind, nb * 64);
L->nb_cap = nb;
}
size_t need = n + 16;
if (L->out_cap < need) {
L->starts = (uint32_t *)realloc(L->starts, need * sizeof(uint32_t));
L->kinds = (uint8_t *)realloc(L->kinds, need);
L->out_cap = need;
}
}
size_t ccc_lex(CccLex *L, const uint8_t *src, size_t n) {
static bool luts = false;
if (!luts) { build_pair_luts(); luts = true; }
if (n == 0) return 0;
size_t nb = (n + 63) / 64;
ccc_ensure(L, n);
classify(src, n, L->word, L->st, L->op, L->kind);
carve(src, n, L->st, L->kind, L->word, L->op);
return compress(L->st, L->kind, nb, L->starts, L->kinds);
}
uint8_t *ccc_read_file(const char *path, size_t *out_len) {
FILE *f = fopen(path, "rb");
if (!f) return NULL;
fseek(f, 0, SEEK_END);
long sz = ftell(f);
fseek(f, 0, SEEK_SET);
if (sz < 0) { fclose(f); return NULL; }
uint8_t *buf = ccc_source_alloc((size_t)sz);
if (!buf) { fclose(f); return NULL; }
if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
fclose(f); ccc_source_free(buf); return NULL;
}
fclose(f);
*out_len = (size_t)sz;
return buf;
}
#ifndef CCC_NO_MAIN
static double now_sec(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9;
}
static const char *base_name(const char *p) {
const char *b = p;
for (const char *q = p; *q; q++)
if (*q == '/' || *q == '\\') b = q + 1;
return b;
}
static void bench_one(CccLex *L, const char *name, const uint8_t *src, size_t len) {
int iters = (len < 50000) ? 200 : 30;
for (int i = 0; i < 3; i++) (void)ccc_lex(L, src, len);
uint64_t t0 = __rdtsc();
double s0 = now_sec();
size_t total = 0;
for (int i = 0; i < iters; i++) total += ccc_lex(L, src, len);
double s1 = now_sec();
uint64_t t1 = __rdtsc();
double bytes = (double)len * iters;
printf(" %-24s %9zu %6.2f %6.2f %9zu\n", name, len,
(double)(t1 - t0) / bytes, bytes / ((s1 - s0) * 1e9), total / (size_t)iters);
}
int main(int argc, char **argv) {
CccLex L; memset(&L, 0, sizeof(L));
if (argc == 3 && !strcmp(argv[1], "--dump")) {
size_t len = 0;
uint8_t *src = ccc_read_file(argv[2], &len);
if (!src) { fprintf(stderr, "could not read %s\n", argv[2]); return 1; }
size_t nt = ccc_lex(&L, src, len);
for (size_t i = 0; i < nt; i++) printf("%u %u\n", L.starts[i], L.kinds[i]);
ccc_source_free(src);
ccc_free(&L);
return 0;
}
if (argc >= 4 && !strcmp(argv[1], "--spin")) {
double secs = atof(argv[2]);
enum { MAXF = 64 };
uint8_t *srcs[MAXF]; size_t lens[MAXF];
int nf = argc - 3;
if (nf > MAXF) nf = MAXF;
for (int i = 0; i < nf; i++) {
srcs[i] = ccc_read_file(argv[3 + i], &lens[i]);
if (!srcs[i]) { fprintf(stderr, "could not read %s\n", argv[3 + i]); return 1; }
}
size_t tok = 0;
for (int i = 0; i < nf; i++) tok += ccc_lex(&L, srcs[i], lens[i]);
double t0 = now_sec();
uint64_t c0 = __rdtsc();
double bytes = 0;
size_t rounds = 0;
while (now_sec() - t0 < secs) {
for (int i = 0; i < nf; i++) { tok += ccc_lex(&L, srcs[i], lens[i]); bytes += (double)lens[i]; }
rounds++;
}
uint64_t c1 = __rdtsc();
double t1 = now_sec();
printf("spin: %zu rounds %.0f bytes %5.2f cyc/B %7.0f MB/s\n",
rounds, bytes, (double)(c1 - c0) / bytes, bytes / ((t1 - t0) * 1e6));
for (int i = 0; i < nf; i++) ccc_source_free(srcs[i]);
ccc_free(&L);
(void)tok;
return 0;
}
if (argc == 1) {
fprintf(stderr,
"usage: %s file.js ... benchmark (cyc/B, GB/s)\n"
" %s --dump file.js print the token stream\n"
" %s --spin S file ... round-robin many files for ~S seconds\n",
argv[0], argv[0], argv[0]);
return 2;
}
printf(" %-24s %9s %6s %6s %9s\n", "file", "bytes", "cyc/B", "GB/s", "tokens");
for (int a = 1; a < argc; a++) {
size_t len = 0;
uint8_t *src = ccc_read_file(argv[a], &len);
if (!src) { fprintf(stderr, " skip %s — could not read\n", argv[a]); continue; }
bench_one(&L, base_name(argv[a]), src, len);
ccc_source_free(src);
}
ccc_free(&L);
return 0;
}
#endif
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment