sedlex-2

tokenizer-expanded.ml

let __sedlex_table_1 =
  "\001\001\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001"
let __sedlex_table_2 =
  "\001\000\000\000\000\000\000\000\000\000\002\002\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\003\003\003\003\003\003\003\003\003\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\004"
let __sedlex_partition_2 =
  function
  | c ->
      if c <= 8
      then (-1)
      else
        if c <= 32 then (Char.code (__sedlex_table_1.[c - 9])) - 1 else (-1)
let __sedlex_partition_1 =
  function
  | c ->
      if c <= 97 then (Char.code (__sedlex_table_2.[c - (-1)])) - 1 else (-1)
let __sedlex_partition_3 =
  function | c -> if c <= 47 then (-1) else if c <= 57 then 0 else (-1)
let __sedlex_partition_4 =
  function | c -> if c <= 96 then (-1) else if c <= 97 then 0 else (-1)
let rec token buf =
  let rec __sedlex_state_0 =
    function
    | buf ->
        (match __sedlex_partition_1 (Sedlexing.next buf) with
         | 0 -> 3
         | 1 -> __sedlex_state_2 buf
         | 2 -> __sedlex_state_3 buf
         | 3 -> __sedlex_state_4 buf
         | _ -> Sedlexing.backtrack buf)
  and __sedlex_state_2 =
    function
    | buf ->
        (Sedlexing.mark buf 1;
         (match __sedlex_partition_2 (Sedlexing.next buf) with
          | 0 -> __sedlex_state_2 buf
          | _ -> Sedlexing.backtrack buf))
  and __sedlex_state_3 =
    function
    | buf ->
        (Sedlexing.mark buf 0;
         (match __sedlex_partition_3 (Sedlexing.next buf) with
          | 0 -> __sedlex_state_3 buf
          | _ -> Sedlexing.backtrack buf))
  and __sedlex_state_4 =
    function
    | buf ->
        (match __sedlex_partition_4 (Sedlexing.next buf) with
         | 0 -> __sedlex_state_5 buf
         | _ -> Sedlexing.backtrack buf)
  and __sedlex_state_5 =
    function
    | buf ->
        (match __sedlex_partition_4 (Sedlexing.next buf) with
         | 0 -> __sedlex_state_6 buf
         | _ -> Sedlexing.backtrack buf)
  and __sedlex_state_6 =
    function
    | buf ->
        (Sedlexing.mark buf 2;
         (match __sedlex_partition_4 (Sedlexing.next buf) with
          | 0 -> __sedlex_state_6 buf
          | _ -> Sedlexing.backtrack buf)) in
  Sedlexing.start buf;
  (match __sedlex_state_0 buf with
   | 0 ->
       (Printf.printf "Number %s\n" (Sedlexing.Latin1.lexeme buf); token buf)
   | 1 -> (Printf.printf "Blank\n"; token buf)
   | 2 -> (Printf.printf "tt\n"; token buf)
   | 3 -> print_endline "EOF"
   | _ -> Printf.printf "Unexpected character\n")
let () =
  let lexbuf = Sedlexing.Latin1.from_string "123 321 456" in token lexbuf

tokenizer.ml

let digit = [%sedlex.regexp? '0'..'9']
let number = [%sedlex.regexp? Plus digit]
let tt = [%sedlex.regexp? Star 'a',Star 'a',Star 'a','a','a','a']

let rec token buf =
  match%sedlex buf with
  | number -> 
     Printf.printf "Number %s\n" (Sedlexing.Latin1.lexeme buf); 
     token buf
  | Plus xml_blank -> Printf.printf "Blank\n"; token buf
  | tt -> Printf.printf "tt\n"; token buf
  | eof -> print_endline "EOF"
  | _ -> Printf.printf "Unexpected character\n"

let () =
  let lexbuf = 
    Sedlexing.Latin1.from_string 
    "123 321 456" in
  token lexbuf