A lexer for the List-to-C compiler

lexer.re

let tokenizer input => {
  let rec tok input current tokens =>
    switch input {
    | [] => List.rev tokens
    | _ =>
      let head = List.hd input;
      let tail = List.tl input;
      let next = tok tail;
      switch (head, current, tokens) {
      /* State: None */
      | ('(', None, t) => next None [OpenParen, ...t]
      | (')', None, t) => next None [CloseParen, ...t]
      | (' ' | '\t' | '\r' | '\n', None, t) => next None t
      | ('"', None, t) => next (Some (String "")) t
      | ('0'..'9' as i, None, t) => next (Some (Number (String.make 1 i))) t
      | ('a'..'z' as i, None, t) => next (Some (Name (String.make 1 i))) t
      /* State: String */
      | ('"', Some (String c), t) => next None [String c, ...t] /* TODO allow escaped double quotes :) */
      | (i, Some (String c), t) => next (Some (String (c ^ String.make 1 i))) t
      /* State: Number */
      | ('0'..'9' as i, Some (Number c), t) => next (Some (Number (c ^ String.make 1 i))) t
      | (')', Some (Number c), t) => next None [CloseParen, Number c, ...t]
      | (' ', Some (Number c), t) => next None [Number c, ...t]
      /* State: Name */
      | ('a'..'z' as i, Some (Name c), t) => next (Some (Name (c ^ String.make 1 i))) t
      | (')', Some (Name c), t) => next None [CloseParen, Name c, ...t]
      | (' ', Some (Name c), t) => next None [Name c, ...t]
      /* Errors */
      | (_, _, t) => List.rev t /* TODO: handle errors */
      }
    };
  tok (explode input) None []
};