sedlex test

output

Ident foobar
Ident A123Bfoo
Op ++
Number 123
Ident Xbar
Op /
Ident foo
EOF

tokenizer.ml

let digit = [%sedlex.regexp? '0'..'9']
let number = [%sedlex.regexp? Plus digit]
let letter = [%sedlex.regexp? 'a'..'z'|'A'..'Z']
let ident = [%sedlex.regexp? letter, Star ('A'..'Z' | 'a'..'z'| digit)]

let rec token buf =
  match%sedlex buf with
  | number -> 
     Printf.printf "Number %s\n" (Sedlexing.Latin1.lexeme buf); 
     token buf
  | ident -> 
     Printf.printf "Ident %s\n" (Sedlexing.Latin1.lexeme buf); 
     token buf
  | Plus xml_blank -> token buf
  | Plus (Chars "+*-/") -> 
     Printf.printf "Op %s\n" (Sedlexing.Latin1.lexeme buf); 
     token buf
  | 128 .. 255 -> print_endline "Non ASCII"
  | eof -> print_endline "EOF"
  | _ -> failwith "Unexpected character"

let () =
  let lexbuf = 
    Sedlexing.Latin1.from_string 
    "foobar A123Bfoo  ++123Xbar/foo" in
  token lexbuf

tokenizer_expanded.ml

let __sedlex_table_2 =
  "\001\001\001\001\001\001\001\001\001\001\000\000\000\000\000\000\000\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\000\000\000\000\000\000\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001\001"
let __sedlex_table_1 =
  "\001\001\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001"
let __sedlex_table_3 = "\001\001\000\001\000\001"
let __sedlex_table_4 =
  "\001\000\000\000\000\000\000\000\000\000\002\002\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\003\003\000\003\000\003\004\004\004\004\004\004\004\004\004\004\000\000\000\000\000\000\000\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\000\000\000\000\000\000\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\005\000\000\000\000\000\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006\006"
let __sedlex_partition_2 =
  function
  | c ->
      if c <= 8
      then (-1)
      else
        if c <= 32 then (Char.code (__sedlex_table_1.[c - 9])) - 1 else (-1)
let __sedlex_partition_4 =
  function | c -> if c <= 47 then (-1) else if c <= 57 then 0 else (-1)
let __sedlex_partition_5 =
  function
  | c ->
      if c <= 47
      then (-1)
      else
        if c <= 122
        then (Char.code (__sedlex_table_2.[c - 48])) - 1
        else (-1)
let __sedlex_partition_3 =
  function
  | c ->
      if c <= 41
      then (-1)
      else
        if c <= 47 then (Char.code (__sedlex_table_3.[c - 42])) - 1 else (-1)
let __sedlex_partition_1 =
  function
  | c ->
      if c <= 255
      then (Char.code (__sedlex_table_4.[c - (-1)])) - 1
      else (-1)
let rec token buf =
  let rec __sedlex_state_0 =
    function
    | buf ->
        (match __sedlex_partition_1 (Sedlexing.next buf) with
         | 0 -> 5
         | 1 -> __sedlex_state_2 buf
         | 2 -> __sedlex_state_3 buf
         | 3 -> __sedlex_state_4 buf
         | 4 -> __sedlex_state_5 buf
         | 5 -> 4
         | _ -> Sedlexing.backtrack buf)
  and __sedlex_state_2 =
    function
    | buf ->
        (Sedlexing.mark buf 2;
         (match __sedlex_partition_2 (Sedlexing.next buf) with
          | 0 -> __sedlex_state_2 buf
          | _ -> Sedlexing.backtrack buf))
  and __sedlex_state_3 =
    function
    | buf ->
        (Sedlexing.mark buf 3;
         (match __sedlex_partition_3 (Sedlexing.next buf) with
          | 0 -> __sedlex_state_3 buf
          | _ -> Sedlexing.backtrack buf))
  and __sedlex_state_4 =
    function
    | buf ->
        (Sedlexing.mark buf 0;
         (match __sedlex_partition_4 (Sedlexing.next buf) with
          | 0 -> __sedlex_state_4 buf
          | _ -> Sedlexing.backtrack buf))
  and __sedlex_state_5 =
    function
    | buf ->
        (Sedlexing.mark buf 1;
         (match __sedlex_partition_5 (Sedlexing.next buf) with
          | 0 -> __sedlex_state_5 buf
          | _ -> Sedlexing.backtrack buf)) in
  Sedlexing.start buf;
  (match __sedlex_state_0 buf with
   | 0 ->
       (Printf.printf "Number %s\n" (Sedlexing.Latin1.lexeme buf); token buf)
   | 1 ->
       (Printf.printf "Ident %s\n" (Sedlexing.Latin1.lexeme buf); token buf)
   | 2 -> token buf
   | 3 -> (Printf.printf "Op %s\n" (Sedlexing.Latin1.lexeme buf); token buf)
   | 4 -> print_endline "Non ASCII"
   | 5 -> print_endline "EOF"
   | _ -> failwith "Unexpected character")
let () =
  let lexbuf = Sedlexing.Latin1.from_string "foobar A123Bfoo  ++123Xbar/foo" in
  token lexbuf

tokenizer_expanded.mli

val __sedlex_table_2 : string
val __sedlex_table_1 : string
val __sedlex_table_3 : string
val __sedlex_table_4 : string
val __sedlex_partition_2 : int -> int
val __sedlex_partition_4 : int -> int
val __sedlex_partition_5 : int -> int
val __sedlex_partition_3 : int -> int
val __sedlex_partition_1 : int -> int
val token : Sedlexing.lexbuf -> unit