Skip to content

Instantly share code, notes, and snippets.

@backupbrain
Last active October 15, 2018 21:49
Show Gist options
  • Save backupbrain/e7e82c7389d3d8df1628c9ee0dfbaf99 to your computer and use it in GitHub Desktop.
Save backupbrain/e7e82c7389d3d8df1628c9ee0dfbaf99 to your computer and use it in GitHub Desktop.
LexicalAnalyzer.prototype.getClassifiedTokenFromLines = function(tokenLines) {
classifiedTokens = [];
errors = [];
// loop through each line of code
for (var lineId = 0; lineId < tokenLines.length; lineId++) {
line = tokenLines[lineId];
// loop through each token in each line
for (var tokenId = 0; tokenId < line.length; tokenId++) {
token = line[tokenId];
classifiedToken = null;
// try to match the token agains all known token types
for (var tokenTypeId in this.tokenTypes) {
tokenType = this.tokenTypes[tokenTypeId];
if (tokenType.test(token) == true) {
classifiedToken = {
"class": tokenTypeId,
"lexeme": token,
"line": lineId
}
classifiedTokens.push(classifiedToken);
break;
}
}
if (classifiedToken == null) {
errors.push(this.generateParseError(lineId, "Unknown token: '" + token + "'"));
}
}
}
output = {
"classifiedTokens": classifiedTokens,
"errors": errors
};
return output;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment