Skip to content

Instantly share code, notes, and snippets.

@The-Quill
Created July 20, 2016 02:00
Show Gist options
  • Save The-Quill/708accffeb08b11d371011cbfecab64e to your computer and use it in GitHub Desktop.
Save The-Quill/708accffeb08b11d371011cbfecab64e to your computer and use it in GitHub Desktop.
A new tokenizer
class Tokenizer {
constructor(code){
this.Code = code.split('')
this.newScopeChars = ['(', '{', '[']
this.scopeChars = {
'(': ')',
'{': '}',
'[': ']'
}
this.evalIndex = 0;
}
start(){
this.evaluate(this.parse(0, null))
}
*getChar (index = 0, end = this.Code.length){
var relativeIndex = index;
for (let code of this.Code.slice(index, end)) {
yield {
code: code,
index: relativeIndex++
};
}
}
isEndScope(char, currentScopeChar){
return (char === this.scopeChars[currentScopeChar])
}
evaluate(...whatever){
console.log(` - eval ${this.evalIndex++}`, ...whatever)
return whatever.map(item => item.code.toString()).join('')
}
handleNewScope(code, char, index){
if (this.newScopeChars.includes(char)){
return this.parse(index + 1, char)
}
return false;
}
parse(startIndex, currentScopeChar){
var buildUp = "";
var codeFunction = this.getChar(startIndex);
var code = codeFunction.next()
var index;
while(!code.done){
index = code.value.index;
var char = code.value.code;
var result = this.handleNewScope(code, char, index);
if (result != false){
buildUp += this.evaluate(result)
codeFunction = this.getChar(result.end + 1)
code = codeFunction.next()
if (code.done) break;
char = code.value.code;
index = code.value.index;
}
if (currentScopeChar != null && this.isEndScope(char, currentScopeChar)){
break;
}
buildUp += char
code = codeFunction.next()
}
return {
code: buildUp,
end: index
}
}
}
var T = new Tokenizer(`some text [sometext, someothertext, whynot] (bracket number 1 (test1 (test2 (test3)))) a different {bracket number 2}`)
T.start()
@vihanb
Copy link

vihanb commented Jul 20, 2016

10/10 for brilliant idea of using generator for getChar

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment