Skip to content

Instantly share code, notes, and snippets.

@nakosung
Last active September 2, 2015 22:41
Show Gist options
  • Save nakosung/8e4dd6c0c593aa3c9567 to your computer and use it in GitHub Desktop.
Save nakosung/8e4dd6c0c593aa3c9567 to your computer and use it in GitHub Desktop.
// non-latent version
function Tokenizer(cb) {
var rules = [];
var buf = '';
function test(buf) {
for (var kk in rules) {
var rule = rules[kk];
if (rule[0].test(buf)) return rule;
}
}
var prev_rule = null;
return {
addRule : function (regex,token) {
rules.push([regex,token]);
},
write : function (str) {
for (var k in str) {
var ch = str[k];
var rule = prev_rule;
if (rule) {
if (rule[0].test(buf+ch)) {
buf += ch;
} else {
cb(buf,{type:prev_rule[1]});
buf = ch;
prev_rule = test(buf);
}
} else {
buf += ch
prev_rule = test(buf);
}
}
},
end : function () {
var rule = test(buf);
if (rule) cb(buf,{type:rule[1]});
}
}
}
function tk(cb) {
var t = new Tokenizer(cb);
t.addRule(/^\/\*([^*]|\*(?!\/))*\*\/$/, 'area comment');
t.addRule(/^\/\*([^*]|\*(?!\/))*\*?$/, 'area comment continue');
t.addRule(/^\/\/[^\n]*$/, 'line comment');
t.addRule(/^"([^"\n]|\\")*"?$/, 'quote');
t.addRule(/^'(\\?[^'\n]|\\')'?$/, 'char');
t.addRule(/^'[^']*$/, 'char continue');
t.addRule(/^#(\S*)$/, 'directive');
t.addRule(/^\($/, 'open paren');
t.addRule(/^\)$/, 'close paren');
t.addRule(/^\[$/, 'open square');
t.addRule(/^\]$/, 'close square');
t.addRule(/^{$/, 'open curly');
t.addRule(/^}$/, 'close curly');
t.addRule(
/^([-<>~!%^&*\/+=?|.,:;]|->|<<|>>|\*\*|\|\||&&|--|\+\+|[-+*|&%\/=]=)$/,
'operator'
);
t.addRule(/^([_A-Za-z]\w*)$/, 'identifier');
t.addRule(/^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$/, 'number');
t.addRule(/^(\s+)$/, 'whitespace');
t.addRule(/^\\\n?$/, 'line continue');
return t;
}
function wrap(source) {
var out = '';
var x = tk(function(src,token){
console.log(src,token)
if (token.type == 'identifier') {
out += "this." + src;
} else {
out += src;
}
});
x.write(source);
x.end();
return out;
}
function make_fn(str) {
return eval("function test() { return " + str + " }; test");
}
var fn = make_fn(wrap("xyz*(x+x)"));
console.log(fn.call({x:4,xyz:2}));
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment