backupbrain

// Use Gists to store code you would like to remember later on
console.log(window); // log the "window" object to the console

backupbrain

Tokenizer.prototype.stripComments = function(text) {
	return text.replace(/\/\/[^\n]+/g, "");
}

backupbrain

Tokenizer.prototype.trimWhitespace = function(text) {
	cleanedText = text.replace(/\n+/g, "\n");
	cleanedText = cleanedText.replace(/[ ]+/g, " ");
	cleanedText = cleanedText.trim();
	return cleanedText;
}

backupbrain

Tokenizer.prototype.resetNewlines = function(text) {
	nlText = text.replace(/\n/g, "").replace(/\;/g, ";\n");
	nlText = nlText.replace(/\}/g, "}\n").replace(/\{/g, "\n{\n");
	nlText = nlText.replace(/\n[ ]+/g, "\n");
	nlText = nlText.replace(/\n$/, "");
	return nlText;
}

backupbrain

Tokenizer.prototype.padTokens = function(text) {
	tokenized = text.replace(/ ?([,\(\)\/\[\]\+\*\-]+) ?/g, " $1 ");
	tokenized = tokenized.replace(/ ?([,]) ?/g, " $1 ");
	tokenized = tokenized.replace(/- >/g, "->")
	tokenized = tokenized.replace(/ ?(==) ?/g, " $1 ");
	tokenized = tokenized.replace(/;/g, " ;");
	tokenized = tokenized.replace(/\s{2,}/g, " ");
	return tokenized;
}

backupbrain

Tokenizer.prototype.getTokenLines = function(text) {
	lines = text.split("\n");
	tokenLines = [];
	for (var lineId = 0; lineId < lines.length; lineId++) {
		line = lines[lineId];
		tokens = line.split(" ");
		tokenLines.push(tokens);
	}
	return tokenLines;
}

backupbrain

Tokenizer.prototype.run = function(text) {
	formattedText = this.prepareCodeForTokenization(text);
	tokens = this.getTokenLines(formattedText);
	return tokens;
}

Tokenizer.prototype.prepareCodeForTokenization = function(text) {
	text = this.stripComments(text);
	text = this.trimWhitespace(text);
	text = this.resetNewlines(text);

backupbrain

function LexicalAnalyzer() {
	this.tokenTypes = {
		"include": /^include$/,
		"string": /^\"[^\"]+\"$/,
		"unaryop": /^(sin|cos|tan|exp|ln|sqrt)$/,
		"exp": /^(\+|\-|\*|\/|\^)$/,
		"comma": /^\,$/,
		"openparen": /^\($/,
		"closeparen": /^\)$/,
		"openbracket": /^\[$/,

backupbrain

LexicalAnalyzer.prototype.getClassifiedTokenFromLines = function(tokenLines) {
	classifiedTokens = [];
	errors = [];
	// loop through each line of code
	for (var lineId = 0; lineId < tokenLines.length; lineId++) {
		line = tokenLines[lineId];
		// loop through each token in each line
		for (var tokenId = 0; tokenId < line.length; tokenId++) {
			token = line[tokenId];
			classifiedToken = null;

backupbrain

LexicalAnalyzer.prototype.run = function(tokenLines) {
	lexOutput = this.getClassifiedTokenFromLines(tokenLines);
	return lexOutput;
};