gistfile1.txt

let ws = [' ' '\t' '\r' '\n']

let digit = ['0'-'9']

let lower = ['a'-'z']
let upper = ['A'-'Z']

let letter = lower | upper

let newline = "\r\n" | ['\r' '\n']

rule token = parse
| eof
    { EOF }
| ws+
    { token lexbuf }
| "{-"
    { blockComment lexbuf;
      token lexbuf }
| "--"
    { lineComment lexbuf;
      token lexbuf }
| '('
    { LPAREN }
| ')'
    { RPAREN }
| '['
    { LBRACKET }
| ']'
    { RBRCAKET }
| '{'
    { LBRACE }
| '}'
    { RBRACE }
| ':'
    { COLON }
| '.'
    { PERIOD }
| ','
    { COMMA }
| "t"
    { BOOL(true) }
| "f"
    { BOOL(false) }
| digit+
    { INT(int_of_string (Lexing.lexeme lexbuf)) }

| digit+ '.' digit+
    { DOUBLE(double_of_string(Lexing.lexeme lexbuf)) }

| digit+ ['e' 'E'] ['+' '-']? digit+
    { DOUBLE(double_of_string(Lexing.lexeme lexbuf)) }

| digit+ '.' digit+ ['f' 'F']
    { let lexeme = Lexing.lexeme lexbuf in
        SINGLE(float_of_string(String.sub(lexeme 0 (String.len(lexeme) - 1)))) }

| digit+ ['e' 'E'] ['+' '-']? digit+ ['f' 'F']
    { let lexeme = Lexing.lexeme lexbuf in
        SINGLE(float_of_string(String.sub(lexeme 0 (String.len(lexeme) - 1)))) }

| '\"'
    { stringLiteral lexbuf }
| '\''
    { letterLiteral lexbuf }

| (letter | '_') (letter | digit | '_')*
    { ID(Lexing.lexeme lexbuf) }

and letterLiteral = parse
| eof
    { failwith
	(Printf.sprintf "unexpected eof.  %d-%d"
	   (Lexing.lexeme_start lexbuf)
	   (Lexing.lexeme_end lexbuf)) }
| newline
    { failwith
	(Printf.sprintf "invalid character \'%s\'. %d-%d"
	   (Lexing.lexeme lexbuf)
	   (Lexing.lexeme_start lexbuf)
	   (Lexing.lexeme_end lexbuf)) }
| _
    { failwith
	(Printf.sprintf "unterminated character-literal. %d-%d"
	   (Lexing.lexeme_start lexbuf)
	   (Lexing.lexeme_end lexbuf)) }

| _ '\''
    { Letter(Lexing.lexeme lexbuf) }

and stringLiteral = parse
| eof
    { failwith
	(Printf.sprintf "unexpected eof.  %d-%d"
	   (Lexing.lexeme_start lexbuf)
	   (Lexing.lexeme_end lexbuf)) }
| newline
    { failwith
	(Printf.sprintf "invalid character \'%s\'. %d-%d"
	   (Lexing.lexeme lexbuf)
	   (Lexing.lexeme_start lexbuf)
	   (Lexing.lexeme_end lexbuf)) }

| '\"'
    { () }

| _
    { stringLiteral lexbuf }

and blockComment = parse
| eof
    { failwith
	(Printf.sprintf "unterminated block-comment. %d-%d"
	   (Lexing.lexeme_start lexbuf)
	   (Lexing.lexeme_end lexbuf)) }
| "-}"
    { () }
| "{-"
    { blockComment lexbuf;
      blockComment lexbuf }
| _
    { blockComment lexbuf}

and lineComment = parse
| ( eof | newline )
    { () }
| _
    { lineComment lexbuf }