Last active
December 18, 2015 07:09
-
-
Save MattDiesel/5744263 to your computer and use it in GitHub Desktop.
First implementation of a simple AutoIt lexer.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #include <Math.au3> | |
| #include <Array.au3> | |
| #include "al_keywords.au3" | |
| #include "al_funcs.au3" | |
| #include "al_macros.au3" | |
| Global Enum $AL_TOK_EOF = 0, _ ; End of File | |
| $AL_TOK_EOL, _ ; End of Line | |
| $AL_TOK_OP, _ ; Operator. | |
| $AL_TOK_ASSIGN, _ ; Operator. | |
| $AL_TOK_KEYWORD, _ ; Keyword. E.g. 'Func', 'Local' etc. (not used if $AL_FLAG_NORESOLVEKEYWORD is set) | |
| $AL_TOK_FUNC, _ ; Standard function (not used if $AL_FLAG_NORESOLVEKEYWORD is set) | |
| $AL_TOK_WORD, _ ; Word (but not keyword or standard function) | |
| $AL_TOK_OPAR, _ ; ( | |
| $AL_TOK_EPAR, _ ; ) | |
| $AL_TOK_OBRACK, _ ; [ | |
| $AL_TOK_EBRACK, _ ; ] | |
| $AL_TOK_COMMA, _ ; , | |
| $AL_TOK_STR, _ ; " ... " | |
| $AL_TOK_NUMBER, _ ; Integer, float, hex etc. | |
| $AL_TOK_MACRO, _ ; @... | |
| $AL_TOK_VARIABLE, _ ; $... | |
| $AL_TOK_PREPROC, _; Preprocessor statement. NB returns whole line and does no processing apart from #cs and #ce | |
| $AL_TOK_COMMENT, _ ; Comment. Includes multiline. | |
| $AL_TOK_LINECONT ; Line continuation _ (only used if $AL_FLAG_AUTOLINECONT not set) | |
| Global Enum Step *2 _ | |
| $AL_FLAG_AUTOLINECONT = 1, _ | |
| $AL_FLAG_NORESOLVEKEYWORD, _ | |
| $AL_FLAG_AUTOINCLUDE, _ | |
| $__AL_FLAG_LINECONT | |
| Global Enum $AL_LEXI_FILENAME = 0, _ | |
| $AL_LEXI_DATA, _ | |
| $AL_LEXI_FLAGS, _ | |
| $AL_LEXI_ABS, _ | |
| $AL_LEXI_LINE, _ | |
| $AL_LEXI_COL, _ | |
| $AL_LEXI_LASTTOK_ABS, _ | |
| $AL_LEXI_LASTTOK_LINE, _ | |
| $AL_LEXI_LASTTOK_COL, _ | |
| $__AL_LEXI_COUNT | |
| Global Enum $AL_ST_START = -1, _ | |
| $AL_ST_NONE, _ | |
| $AL_ST_INT, _ | |
| $AL_ST_FLOAT, _ | |
| $AL_ST_FLOATE, _ | |
| $AL_ST_FLOATES, _ | |
| $AL_ST_ZERO, _ | |
| $AL_ST_HEX, _ | |
| $AL_ST_STRINGS, _ | |
| $AL_ST_STRINGD, _ | |
| $AL_ST_MACRO, _ | |
| $AL_ST_VARIABLE, _ | |
| $AL_ST_COMMENT, _ | |
| $AL_ST_COMMENTMULTI, _ | |
| $AL_ST_COMMENTMULTINL, _ | |
| $AL_ST_COMMENTMULTIEND, _ | |
| $AL_ST_PREPROC, _ | |
| $AL_ST_PREPROCLINE, _ | |
| $AL_ST_INCLUDELINE, _ | |
| $AL_ST_LINECONT, _ | |
| $AL_ST_KEYWORD | |
| ; Test assertions: | |
| ;~ _AuLex_LexTestAssert("", $AL_TOK_EOF, "") | |
| ;~ _AuLex_LexTestAssert(@CRLF, $AL_TOK_EOL, @CRLF) | |
| ;~ _AuLex_LexTestAssert(@LF, $AL_TOK_EOL, @LF) | |
| ;~ _AuLex_LexTestAssert(@CR, $AL_TOK_EOL, @CR) | |
| ;~ _AuLex_LexTestAssert("123", $AL_TOK_NUMBER, "123") | |
| ;~ _AuLex_LexTestAssert("123.456", $AL_TOK_NUMBER, "123.456") | |
| ;~ _AuLex_LexTestAssert("123e4", $AL_TOK_NUMBER, "123e4") | |
| ;~ _AuLex_LexTestAssert("123e-4", $AL_TOK_NUMBER, "123e-4") | |
| ;~ _AuLex_LexTestAssert("123.456e4", $AL_TOK_NUMBER, "123.456e4") | |
| ;~ _AuLex_LexTestAssert("123.456e-4", $AL_TOK_NUMBER, "123.456e-4") | |
| ;~ _AuLex_LexTestAssert("0111", $AL_TOK_NUMBER, "0111") | |
| ;~ _AuLex_LexTestAssert("0xABCDEF", $AL_TOK_NUMBER, "0xABCDEF") | |
| ;~ _AuLex_LexTestAssert("'This is a test'", $AL_TOK_STR, "'This is a test'") | |
| ;~ _AuLex_LexTestAssert("'This is '' a test'", $AL_TOK_STR, "'This is '' a test'") | |
| ;~ _AuLex_LexTestAssert("""This is a test""", $AL_TOK_STR, """This is a test""") | |
| ;~ _AuLex_LexTestAssert("""This is """" a test""", $AL_TOK_STR, """This is """" a test""") | |
| ;~ _AuLex_LexTestAssert("@Testing", $AL_TOK_MACRO, "@Testing") | |
| ;~ _AuLex_LexTestAssert("$Testing", $AL_TOK_VARIABLE, "$Testing") | |
| ;~ _AuLex_LexTestAssert("; This is a comment", $AL_TOK_COMMENT, "; This is a comment") | |
| ;~ _AuLex_LexTestAssert("#include <Test>", $AL_TOK_PREPROC, "#include <Test>") | |
| ;~ _AuLex_LexTestAssert("#cs" & @CRLF & "Testing comment" & @CRLF & "#This won't match" & @CRLF & "#ce", $AL_TOK_COMMENT) | |
| ;~ _AuLex_LexTestAssert("(", $AL_TOK_OPAR) | |
| ;~ _AuLex_LexTestAssert(")", $AL_TOK_EPAR) | |
| ;~ _AuLex_LexTestAssert("[", $AL_TOK_OBRACK) | |
| ;~ _AuLex_LexTestAssert("]", $AL_TOK_EBRACK) | |
| ;~ _AuLex_LexTestAssert("*", $AL_TOK_OP) | |
| ;~ _AuLex_LexTestAssert("*=", $AL_TOK_OP) | |
| ;~ _AuLex_LexTestAssert("<=", $AL_TOK_OP) | |
| ;~ _AuLex_LexTestAssert("<>", $AL_TOK_OP) | |
| ;~ _AuLex_LexTestAssert("^", $AL_TOK_OP) | |
| ;~ _AuLex_LexTestAssert("_Test", $AL_TOK_WORD) | |
| ;~ _AuLex_LexTestAssert("Test", $AL_TOK_WORD) | |
| ;~ _AuLex_LexTestAssert("Local", $AL_TOK_KEYWORD) | |
| ;~ _AuLex_LexTestAssert("StringInStr", $AL_TOK_FUNC) | |
| ;~ _AuLex_LexTestAssert("_" & @CRLF & " _Test", $AL_TOK_LINECONT, "_", 0) | |
| ;~ _AuLex_LexTestAssert("_" & @CRLF & " _Test", $AL_TOK_WORD, "_Test", $AL_FLAG_AUTOLINECONT) | |
| ;~ _AuLex_LexTestAssert("_ ; Testing Comment" & @CRLF & " _Test", $AL_TOK_COMMENT, "; Testing Comment", $AL_FLAG_AUTOLINECONT) | |
| ;~ | |
| ;~ Func _AuLex_LexTestAssert($sString, $iExpType, $sExpectData = Default, $iFlags = 0) | |
| ;~ If $sExpectData = Default Then $sExpectData = $sString | |
| ;~ | |
| ;~ Local $l = _AuLex_StartLexS("Test", $sString, $iFlags) | |
| ;~ | |
| ;~ Local $sData = _AuLex_NextToken($l) | |
| ;~ Local $iType = @extended | |
| ;~ | |
| ;~ If $iExpType <> $iType Or $sData <> $sExpectData Then | |
| ;~ ConsoleWrite("Assertion Failed! " & @error & @LF & @TAB & "Input: {" & $sString & "} (" & StringToBinary($sData) & ")" & @LF & @TAB & _ | |
| ;~ "Token Type: " & $iType & " (" & $iExpType & ")" & @LF & @TAB & "Token Data: {" & $sData & "} (" & StringToBinary($sData) & ")" & @LF) | |
| ;~ EndIf | |
| ;~ EndFunc ;==>_AuLex_LexTestAssert | |
| ; Example: Process this file: | |
| ;~ Local $l = _AuLex_StartLex(@ScriptName, $AL_FLAG_AUTOLINECONT) | |
| ;~ Local $sData, $iType | |
| ;~ Do | |
| ;~ $sData = _AuLex_NextToken($l) | |
| ;~ $iType = @extended | |
| ;~ ConsoleWrite(StringFormat("%-4.4i: %s\n", $iType, $sData)) | |
| ;~ Until $iType = $AL_TOK_EOF | |
| Func _AuLex_StartLex($sFile, $iFlags) | |
| Local $sData = FileRead($sFile) | |
| If @error Then Return SetError(1, 0, 0) ; File couldn't be read. | |
| Return _AuLex_StartLexS($sFile, $sData, $iFlags) | |
| EndFunc ;==>_AuLex_StartLex | |
| ; Starts a lexer for a data string. | |
| Func _AuLex_StartLexS($sName, $sData, $iFlags) | |
| Local $lexRet[$__AL_LEXI_COUNT] | |
| $lexRet[$AL_LEXI_FILENAME] = $sName | |
| $lexRet[$AL_LEXI_DATA] = $sData & @CRLF | |
| $lexRet[$AL_LEXI_FLAGS] = $iFlags | |
| $lexRet[$AL_LEXI_ABS] = 1 | |
| $lexRet[$AL_LEXI_LINE] = 1 | |
| $lexRet[$AL_LEXI_COL] = 1 | |
| $lexRet[$AL_LEXI_LASTTOK_ABS] = 1 | |
| $lexRet[$AL_LEXI_LASTTOK_LINE] = 1 | |
| $lexRet[$AL_LEXI_LASTTOK_COL] = 1 | |
| Return $lexRet | |
| EndFunc ;==>_AuLex_StartLexS | |
| ; Returns the next token (as a string) | |
| ; The token type (an $AL_TOK_ constant) is returned in @extended | |
| Func _AuLex_NextToken(ByRef $lex) | |
| Local $iState = $AL_ST_START | |
| Local $c, $c2, $anchor | |
| Local $iRetTok = 0 | |
| Local $vRetData = "" | |
| While 1 | |
| $c = __AuLex_NextChar($lex) | |
| Switch $iState | |
| Case $AL_ST_START | |
| Select | |
| Case $c = "" | |
| Return SetExtended($AL_TOK_EOF, "") | |
| Case _String_IsNewLine($c) | |
| If BitAND($lex[$AL_LEXI_FLAGS], $__AL_FLAG_LINECONT) Then | |
| $lex[$AL_LEXI_FLAGS] = BitXOR($lex[$AL_LEXI_FLAGS], $__AL_FLAG_LINECONT) | |
| Else | |
| $lex[$AL_LEXI_LASTTOK_ABS] = $lex[$AL_LEXI_ABS] - StringLen($c) | |
| $lex[$AL_LEXI_LASTTOK_LINE] = $lex[$AL_LEXI_LINE] - 1 | |
| $lex[$AL_LEXI_LASTTOK_COL] = -1 ; TODO | |
| Return SetExtended($AL_TOK_EOL, $c) | |
| EndIf | |
| Case Not StringIsSpace($c) | |
| ; Save token position | |
| $lex[$AL_LEXI_LASTTOK_ABS] = $lex[$AL_LEXI_ABS] - 1 | |
| $lex[$AL_LEXI_LASTTOK_LINE] = $lex[$AL_LEXI_LINE] | |
| $lex[$AL_LEXI_LASTTOK_COL] = $lex[$AL_LEXI_COL] - 1 | |
| $iState = $AL_ST_NONE | |
| __AuLex_PrevChar($lex) | |
| EndSelect | |
| Case $AL_ST_NONE | |
| Select | |
| Case $c = '0' | |
| $iState = $AL_ST_ZERO | |
| Case $c = "'" | |
| $iState = $AL_ST_STRINGS | |
| Case $c = '"' | |
| $iState = $AL_ST_STRINGD | |
| Case $c = "@" | |
| $iState = $AL_ST_MACRO | |
| Case $c = "$" | |
| $iState = $AL_ST_VARIABLE | |
| Case $c = ";" | |
| $iState = $AL_ST_COMMENT | |
| Case $c = "#" | |
| $iState = $AL_ST_PREPROC | |
| Case $c = "(" | |
| $iRetTok = $AL_TOK_OPAR | |
| ExitLoop | |
| Case $c = ")" | |
| $iRetTok = $AL_TOK_EPAR | |
| ExitLoop | |
| Case $c = "[" | |
| $iRetTok = $AL_TOK_OBRACK | |
| ExitLoop | |
| Case $c = "]" | |
| $iRetTok = $AL_TOK_EBRACK | |
| ExitLoop | |
| Case $c = "," | |
| $iRetTok = $AL_TOK_COMMA | |
| ExitLoop | |
| Case StringInStr("*/+-&", $c) | |
| If __AuLex_PeekChar($lex) = "=" Then | |
| __AuLex_NextChar($lex) | |
| $iRetTok = $AL_TOK_ASSIGN | |
| Else | |
| $iRetTok = $AL_TOK_OP | |
| EndIf | |
| ExitLoop | |
| Case StringInStr("=>", $c) | |
| If __AuLex_PeekChar($lex) = "=" Then | |
| __AuLex_NextChar($lex) | |
| EndIf | |
| $iRetTok = $AL_TOK_OP | |
| ExitLoop | |
| Case $c = "<" | |
| If StringInStr("=>", __AuLex_PeekChar($lex)) Then | |
| __AuLex_NextChar($lex) | |
| EndIf | |
| $iRetTok = $AL_TOK_OP | |
| ExitLoop | |
| Case $c = "^" | |
| $iRetTok = $AL_TOK_OP | |
| ExitLoop | |
| Case $c = "_" | |
| $c2 = __AuLex_PeekChar($lex) | |
| If StringIsAlNum($c2) Or $c2 = "_" Then | |
| $iState = $AL_ST_KEYWORD | |
| ElseIf BitAND($lex[$AL_LEXI_FLAGS], $AL_FLAG_AUTOLINECONT) Then | |
| $iState = $AL_ST_LINECONT | |
| Else | |
| $iRetTok = $AL_TOK_LINECONT | |
| ExitLoop | |
| EndIf | |
| Case StringIsDigit($c) | |
| $iState = $AL_ST_INT | |
| Case StringIsAlpha($c) | |
| $iState = $AL_ST_KEYWORD | |
| Case Else | |
| ; ERROR: Invalid character | |
| Return SetError(@ScriptLineNumber, 0, 0) | |
| EndSelect | |
| Case $AL_ST_INT | |
| If $c = '.' Then | |
| $iState = $AL_ST_FLOAT | |
| ElseIf $c = 'e' Then | |
| $iState = $AL_ST_FLOATE | |
| ElseIf Not StringIsDigit($c) Then | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_NUMBER | |
| ExitLoop | |
| EndIf | |
| Case $AL_ST_FLOAT | |
| If $c = 'e' Then | |
| $iState = $AL_ST_FLOATE | |
| ElseIf Not StringIsDigit($c) Then | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_NUMBER | |
| ExitLoop | |
| EndIf | |
| Case $AL_ST_FLOATE | |
| If $c = '+' Or $c = '-' Or StringIsDigit($c) Then | |
| $iState = $AL_ST_FLOATES | |
| Else | |
| __AuLex_PrevChar($lex) | |
| ; NB: Next token will be 'e' which is most likely an error. | |
| $iRetTok = $AL_TOK_NUMBER | |
| ExitLoop | |
| EndIf | |
| Case $AL_ST_FLOATES | |
| If Not StringIsDigit($c) Then | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_NUMBER | |
| ExitLoop | |
| EndIf | |
| Case $AL_ST_ZERO | |
| If StringInStr("Xx", $c) Then | |
| $iState = $AL_ST_HEX | |
| ElseIf $c = '.' Then | |
| $iState = $AL_ST_FLOAT | |
| ElseIf StringIsDigit($c) Then | |
| $iState = $AL_ST_INT | |
| Else | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_NUMBER | |
| ExitLoop | |
| EndIf | |
| Case $AL_ST_HEX | |
| If Not StringIsXDigit($c) Then | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_NUMBER | |
| ExitLoop | |
| EndIf | |
| Case $AL_ST_STRINGS | |
| If $c = "'" Then | |
| $c2 = __AuLex_PeekChar($lex) | |
| If $c2 <> "'" Then | |
| $iRetTok = $AL_TOK_STR | |
| ExitLoop | |
| Else | |
| __AuLex_NextChar($lex) | |
| EndIf | |
| ElseIf $c = "" Or $c = @CRLF Then | |
| ; ERROR: String not terminated | |
| Return SetError(@ScriptLineNumber, 0, 0) | |
| EndIf | |
| Case $AL_ST_STRINGD | |
| If $c = '"' Then | |
| $c2 = __AuLex_PeekChar($lex) | |
| If $c2 <> '"' Then | |
| $iRetTok = $AL_TOK_STR | |
| ExitLoop | |
| Else | |
| __AuLex_NextChar($lex) | |
| EndIf | |
| ElseIf $c = "" Or $c = @CRLF Then | |
| ; ERROR: String not terminated | |
| Return SetError(@ScriptLineNumber, 0, 0) | |
| EndIf | |
| Case $AL_ST_MACRO | |
| If $c <> "_" And Not StringIsAlNum($c) Then | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_MACRO | |
| ExitLoop | |
| EndIf | |
| Case $AL_ST_VARIABLE | |
| If $c <> "_" And Not StringIsAlNum($c) Then | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_VARIABLE | |
| ExitLoop | |
| EndIf | |
| Case $AL_ST_COMMENT | |
| If $c = "" Or _String_IsNewLine($c) Then | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_COMMENT | |
| ExitLoop | |
| EndIf | |
| Case $AL_ST_COMMENTMULTI | |
| If _String_IsNewLine($c) Then | |
| $iState = $AL_ST_COMMENTMULTINL | |
| ElseIf $c = "" Then | |
| ; ERROR: Multiline comment not terminated | |
| Return SetError(@ScriptLineNumber, 0, 0) | |
| EndIf | |
| Case $AL_ST_COMMENTMULTINL | |
| If $c = "#" Then | |
| $iState = $AL_ST_COMMENTMULTIEND | |
| $anchor = $lex[$AL_LEXI_ABS] | |
| ElseIf $c = "" Then | |
| ; ERROR: Multiline comment not terminated | |
| Return SetError(@ScriptLineNumber, 0, 0) | |
| ElseIf _String_IsNewLine($c) Then | |
| $iState = $AL_ST_COMMENTMULTINL | |
| Else | |
| $iState = $AL_ST_COMMENTMULTI | |
| EndIf | |
| Case $AL_ST_COMMENTMULTIEND | |
| If StringIsSpace($c) Or $c = "" Then | |
| Switch StringStripWS(StringMid($lex[$AL_LEXI_DATA], $anchor, $lex[$AL_LEXI_ABS] - $anchor), 2) | |
| Case "ce", "comments-end" | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_COMMENT | |
| ExitLoop | |
| Case Else | |
| If _String_IsNewLine($c) Then | |
| $iState = $AL_ST_COMMENTMULTINL | |
| Else | |
| $iState = $AL_ST_COMMENTMULTI | |
| EndIf | |
| EndSwitch | |
| EndIf | |
| Case $AL_ST_PREPROC | |
| If StringIsSpace($c) Or $c = "" Then | |
| Switch StringStripWS(StringMid($lex[$AL_LEXI_DATA], $lex[$AL_LEXI_LASTTOK_ABS], $lex[$AL_LEXI_ABS] - $lex[$AL_LEXI_LASTTOK_ABS]), 2) | |
| Case "#cs", "#comments-start" | |
| $iState = $AL_ST_COMMENTMULTI | |
| Case "#include" | |
| If Not BitAND($lex[$AL_LEXI_FLAGS], $AL_FLAG_AUTOINCLUDE) Then ContinueCase | |
| $iState = $AL_ST_INCLUDELINE | |
| Case Else | |
| If _String_IsNewLine($c) Then | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_PREPROC | |
| ExitLoop | |
| Else | |
| $iState = $AL_ST_PREPROCLINE | |
| EndIf | |
| EndSwitch | |
| EndIf | |
| Case $AL_ST_PREPROCLINE | |
| If _String_IsNewLine($c) Or $c = "" Then | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_PREPROC | |
| ExitLoop | |
| EndIf | |
| Case $AL_ST_INCLUDELINE | |
| If _String_IsNewLine($c) Or $c = "" Then | |
| $vRetData = StringMid($lex[$AL_LEXI_DATA], _ | |
| $lex[$AL_LEXI_LASTTOK_ABS], $lex[$AL_LEXI_ABS] - $lex[$AL_LEXI_LASTTOK_ABS]) | |
| $c2 = StringStripWS(StringTrimLeft(StringStripWS($vRetData, 3), StringLen("#include")), 3) | |
| Switch StringLeft($c2, 1) | |
| Case '"' | |
| If StringRight($c2, 1) <> '"' Then | |
| ; ERROR: Incorrect include line | |
| Return SetError(@ScriptLineNumber, 0, "") | |
| EndIf | |
| $c2 = StringTrimLeft(StringTrimRight($c2, 1), 1) | |
| ; TODO | |
| Case '<' | |
| If StringRight($c2, 1) <> '>' Then | |
| ; ERROR: Incorrect include line | |
| Return SetError(@ScriptLineNumber, 0, "") | |
| EndIf | |
| $c2 = StringTrimLeft(StringTrimRight($c2, 1), 1) | |
| ; TODO | |
| Case Else | |
| ; ERROR: Incorrect include line | |
| Return SetError(@ScriptLineNumber, 0, "") | |
| EndSwitch | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_PREPROC | |
| ExitLoop | |
| EndIf | |
| Case $AL_ST_LINECONT | |
| If $c = ";" Then | |
| $lex[$AL_LEXI_LASTTOK_ABS] = $lex[$AL_LEXI_ABS] - 1 | |
| $lex[$AL_LEXI_LASTTOK_LINE] = $lex[$AL_LEXI_LINE] | |
| $lex[$AL_LEXI_LASTTOK_COL] = $lex[$AL_LEXI_COL] - 1 | |
| $iState = $AL_ST_COMMENT | |
| $lex[$AL_LEXI_FLAGS] = BitOR($lex[$AL_LEXI_FLAGS], $__AL_FLAG_LINECONT) | |
| ElseIf _String_IsNewLine($c) Then | |
| $iState = $AL_ST_START | |
| ElseIf $c = "" Then | |
| ; Error: No line after a continuation | |
| Return SetError(@ScriptLineNumber, 0, 0) | |
| ElseIf Not StringIsSpace($c) Then | |
| ; ERROR: Something after a line continuation | |
| Return SetError(@ScriptLineNumber, 0, 0) | |
| EndIf | |
| Case $AL_ST_KEYWORD | |
| If Not (StringIsAlNum($c) Or $c = "_") Then | |
| __AuLex_PrevChar($lex) | |
| $iRetTok = $AL_TOK_WORD | |
| ExitLoop | |
| EndIf | |
| EndSwitch | |
| WEnd | |
| $vRetData = StringMid($lex[$AL_LEXI_DATA], _ | |
| $lex[$AL_LEXI_LASTTOK_ABS], $lex[$AL_LEXI_ABS] - $lex[$AL_LEXI_LASTTOK_ABS]) | |
| $vRetData = StringStripWS($vRetData, 3) | |
| If $iRetTok = $AL_TOK_WORD And _ | |
| Not BitAND($lex[$AL_LEXI_FLAGS], $AL_FLAG_NORESOLVEKEYWORD) Then | |
| If _AuLex_IsKeyword($vRetData) Then | |
| $iRetTok = $AL_TOK_KEYWORD | |
| ElseIf _AuLex_IsStandardFunc($vRetData) Then | |
| $iRetTok = $AL_TOK_FUNC | |
| EndIf | |
| EndIf | |
| Return SetExtended($iRetTok, $vRetData) | |
| EndFunc ;==>_AuLex_NextToken | |
| Func _AuLex_IsKeyword($s) | |
| Return _ArrayBinarySearch($__AL_KEYWORDS, $s) >= 0 | |
| EndFunc ;==>_AuLex_IsKeyword | |
| Func _AuLex_IsStandardFunc($s) | |
| Return _ArrayBinarySearch($__AL_FUNCS, $s) >= 0 | |
| EndFunc ;==>_AuLex_IsStandardFunc | |
| ; Returns the next character and increments the counters. | |
| Func __AuLex_NextChar(ByRef $lex) | |
| Local $ret = __AuLex_PeekChar($lex) | |
| If $ret = "" Then Return "" | |
| $lex[$AL_LEXI_ABS] += StringLen($ret) | |
| If _String_IsNewLine($ret) Then | |
| $lex[$AL_LEXI_LINE] += 1 | |
| $lex[$AL_LEXI_COL] = 1 | |
| Else | |
| $lex[$AL_LEXI_COL] += StringLen($ret) | |
| EndIf | |
| Return $ret | |
| EndFunc ;==>__AuLex_NextChar | |
| Func __AuLex_PrevChar(ByRef $lex) | |
| If $lex[$AL_LEXI_ABS] >= StringLen($lex[$AL_LEXI_DATA]) Then Return "" ; Don't step back from EOF | |
| Local $ret = StringMid($lex[$AL_LEXI_DATA], $lex[$AL_LEXI_ABS] - 1, 1) | |
| If $ret = @LF Then | |
| Local $r2 = StringMid($lex[$AL_LEXI_DATA], $lex[$AL_LEXI_ABS] - 2, 1) | |
| If $r2 = @CR Then $ret = @CRLF | |
| EndIf | |
| $lex[$AL_LEXI_ABS] -= StringLen($ret) | |
| If _String_IsNewLine($ret) Then | |
| $lex[$AL_LEXI_LINE] -= 1 | |
| $lex[$AL_LEXI_COL] = $lex[$AL_LEXI_ABS] - _ | |
| _Max(StringInStr($lex[$AL_LEXI_DATA], @LF, 2, -1, $lex[$AL_LEXI_ABS], $lex[$AL_LEXI_ABS]), _ | |
| StringInStr($lex[$AL_LEXI_DATA], @CR, 2, -1, $lex[$AL_LEXI_ABS], $lex[$AL_LEXI_ABS])) | |
| Else | |
| $lex[$AL_LEXI_COL] -= StringLen($ret) | |
| EndIf | |
| Return $ret | |
| EndFunc ;==>__AuLex_PrevChar | |
| ; Returns the next character without incrementing any of the counters. | |
| Func __AuLex_PeekChar(ByRef $lex) | |
| Local $ret = StringMid($lex[$AL_LEXI_DATA], $lex[$AL_LEXI_ABS], 1) | |
| If $ret = @CR Then | |
| Local $r2 = StringMid($lex[$AL_LEXI_DATA], $lex[$AL_LEXI_ABS] + 1, 1) | |
| If $r2 = @LF Then $ret = @CRLF | |
| EndIf | |
| Return $ret | |
| EndFunc ;==>__AuLex_PeekChar | |
| Func _String_IsNewLine($c) | |
| Return StringInStr(@CRLF, $c) | |
| EndFunc ;==>_String_IsNewLine |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment