half finished lexer

lexer.qut

type TokenType =
    | COMMENT
    | IDENTIFIER
    | INTEGER
    | REAL
    | CHARACTER
    | STRING
    | LEFT_BRACE
    | LEFT_BRACKET
    | LEFT_PAREN
    | RIGHT_BRACE
    | RIGHT_BRACKET
    | RIGHT_PAREN
    | DOT
    | COMMA
    | SEMICOLON
    | EQUAL
    | BANG_EQUAL
    | EQUAL_EQUAL
    | GREATER
    | GREATER_EQUAL
    | LESS
    | LESS_EQUAL
    | ASTERISK
    | MINUS
    | PLUS
    | SLASH
    | INVALID
    | EOF

type Span = {
    spos : (Nat, Nat),
    epos : (Nat, Nat),
}

type Token = {
    ty : TokenType,
    span : Span,
    lexeme : TextString,
}

let show @ Show Token
    each field in self  =>  field@show
    self                =>  format "Token({:, })" (each field in self)


type Tokenizer = {
    source : TextString,
    mutable start : Nat,
    mutable current : Nat,
    mutable line_no : Nat,
    mutable tokens : List Token,
}

let new @ Buildable (Type Tokenizer) : TextString -> Tokenizer
    source  =>  {
                    source,
                    start = 0,
                    current = 0,
                    line_no = 1,
                    tokens = new List (),
                }

let is_eof @ Tokenizer
    self    =>  self.current >= self.source.length

let peek @ Tokenizer
    self -> (self@is_eof)       =>  '\0'
    self                        =>  self.source@at self.current

let advance @ Tokenizer
    self    =>  self.current <- mutate (fun \ field => field + 1)

let consume @ Tokenizer
    self    =>  let char = self@peek
                then self@advance
                then char

let munch @ Tokenizer
    : Tokenizer -> Char -> TokenType -> TokenType -> TokenType
    self -> next_char -> token_type -> fallback     =>
        let inner : (eq = next_char | _ : Char) -> TokenType
            eq  => self@advance then token_type
            _   => fallback
        then _inner (self@peek)

# TODO: make_number, make_identifier, make_char, make_string

let get_span @ Tokenizer
    self    =>  { spos = (self.spos, self.line_no), epos = (self.epos, self.line_no) }

let build_token @ Tokenizer
    (self, token_type)  =>  let span = self@get_span
                            then let lexeme = self.source@slice self.start self.current
                            then { ty = token_type, span, lexeme }

let add_token @ Tokenizer
    (self, token_type)  =>  self.tokens <- mutate (fun
                                field => field@append (self@build_token token_type)
                            )

let scan_token @ Tokenizer
    self    =>  let char = self@consume
                then let make_token_type : (
                    | single_quote = '\''
                    | double_quote = '"'
                    | left_brace = '{'
                    | left_bracket = '['
                    | left_paren = '('
                    | right_brace = '}'
                    | right_bracket = ']'
                    | right_paren = ')'
                    | dot = '.'
                    | comma = ','
                    | semicolon = ';'
                    | equal = '='
                    | bang = '!'
                    | greater = '>'
                    | less = '<'
                    | asterisk = '*'
                    | minus = '-'
                    | plus = '+'
                    | slash = '/'
                    | else : Char
                ) -> Option TokenType
                    single_quote                        =>  self@make_char
                    double_quote                        =>  self@make_string
                    left_brace                          =>  some LEFT_BRACE
                    left_bracket                        =>  some LEFT_BRACKET
                    left_paren                          =>  some LEFT_PAREN
                    right_brace                         =>  some RIGHT_BRACE
                    right_bracket                       =>  some RIGHT_BRACKET
                    right_paren                         =>  some RIGHT_PAREN
                    dot                                 =>  some DOT
                    comma                               =>  some COMMA
                    semicolon                           =>  some SEMICOLON
                    equal                               =>  some (self@munch '=' EQUAL_EQUAL EQUAL)
                    bang                                =>  some (self@munch '=' BANG_EQUAL EQUAL)
                    greater                             =>  some (self@munch '=' GREATER_EQUAL GREATER)
                    less                                =>  some (self@munch '=' LESS_EQUAL LESS)
                    asterisk                            =>  some ASTERISK
                    minus                               =>  some MINUS
                    plus                                =>  some PLUS
                    slash                               =>  some SLASH
                    else -> (else@match "\d")           =>  self@make_number
                    else -> (else@match "[A-Za-z_]")    =>  self@make_identifier
                    _                                   =>  some INVALID
                then (make_token_type char)@map (self@add_token)

let scan @ Tokenizer
    each self -> (not (self@is_eof))    =>  self.start <- mutate (fun \ _ => self.current)
                                            then self@scan_token

let run @ Tokenizer
    self    =>  self@scan
                then self@add_token EOF
                then self.tokens


let main : IO
    io  =>  let tokenizer = Tokenizer@new "hello"
            then let tokens = tokenizer@run
            then io.stdout <- writeln (show tokens)