tokenize.lisp

(defun tokenize (s &key emit-fn)
  (let ((lexer-fn (lexer s)))
    (loop :with token :and val
          :do (setf (values token val) (funcall lexer-fn))
          :while (not (null token))
          :if (null emit-fn)
            :collect (cons token val)
          :else
            :do (funcall emit-fn token val))))