Created
July 20, 2016 02:00
-
-
Save The-Quill/708accffeb08b11d371011cbfecab64e to your computer and use it in GitHub Desktop.
A new tokenizer
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Tokenizer { | |
constructor(code){ | |
this.Code = code.split('') | |
this.newScopeChars = ['(', '{', '['] | |
this.scopeChars = { | |
'(': ')', | |
'{': '}', | |
'[': ']' | |
} | |
this.evalIndex = 0; | |
} | |
start(){ | |
this.evaluate(this.parse(0, null)) | |
} | |
*getChar (index = 0, end = this.Code.length){ | |
var relativeIndex = index; | |
for (let code of this.Code.slice(index, end)) { | |
yield { | |
code: code, | |
index: relativeIndex++ | |
}; | |
} | |
} | |
isEndScope(char, currentScopeChar){ | |
return (char === this.scopeChars[currentScopeChar]) | |
} | |
evaluate(...whatever){ | |
console.log(` - eval ${this.evalIndex++}`, ...whatever) | |
return whatever.map(item => item.code.toString()).join('') | |
} | |
handleNewScope(code, char, index){ | |
if (this.newScopeChars.includes(char)){ | |
return this.parse(index + 1, char) | |
} | |
return false; | |
} | |
parse(startIndex, currentScopeChar){ | |
var buildUp = ""; | |
var codeFunction = this.getChar(startIndex); | |
var code = codeFunction.next() | |
var index; | |
while(!code.done){ | |
index = code.value.index; | |
var char = code.value.code; | |
var result = this.handleNewScope(code, char, index); | |
if (result != false){ | |
buildUp += this.evaluate(result) | |
codeFunction = this.getChar(result.end + 1) | |
code = codeFunction.next() | |
if (code.done) break; | |
char = code.value.code; | |
index = code.value.index; | |
} | |
if (currentScopeChar != null && this.isEndScope(char, currentScopeChar)){ | |
break; | |
} | |
buildUp += char | |
code = codeFunction.next() | |
} | |
return { | |
code: buildUp, | |
end: index | |
} | |
} | |
} | |
var T = new Tokenizer(`some text [sometext, someothertext, whynot] (bracket number 1 (test1 (test2 (test3)))) a different {bracket number 2}`) | |
T.start() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
10/10 for brilliant idea of using generator for
getChar