Skip to content

Instantly share code, notes, and snippets.

@wilyJ80
Last active February 16, 2025 21:00
Show Gist options
  • Save wilyJ80/ad77e94f70b4824f89885bcb17743007 to your computer and use it in GitHub Desktop.
Save wilyJ80/ad77e94f70b4824f89885bcb17743007 to your computer and use it in GitHub Desktop.
JS class-based table-driven lexer for a simple Forth
import * as readline from 'node:readline/promises';
import { stdin as input, stdout as output } from 'node:process';
/* valid tokens:
numbers (consisting of digits),
words (consisting of alnum chars),
comment (ignore)
*/
const Transition = class {
#nextState;
#callback;
#category;
#isOther;
constructor(nextState, callback,
category, isOther) {
this.#nextState = nextState;
this.#callback = callback;
this.#category = category;
this.#isOther = isOther;
}
get nextState() {
return this.#nextState;
}
get callback() {
return this.#callback;
}
get category() {
return this.#category;
}
get other() {
return this.#isOther;
}
}
const CharTester = class {
isLetter(ch) {
return new RegExp('[a-z]').test(ch);
}
isDigit(ch) {
return new RegExp('[0-9]').test(ch);
}
isNotLetterNorDigit(ch) {
return !(
new RegExp('[a-z]').test(ch) ||
new RegExp('[0-9]').test(ch)
);
}
isDigitOrLetter(ch) {
return new RegExp('[a-z]|[0-9]').test(ch);
}
isParenOpen(ch) {
return new RegExp('\(').test(ch);
}
isParenClose(ch) {
return new RegExp('\)').test(ch);
}
isNotParenClose(ch) {
return new RegExp('^)').test(ch);
}
isBlank(ch) {
return new RegExp('\s').test(ch);
}
}
const Transitions = class {
#transitionTable;
#tester;
get table() {
return this.#transitionTable;
}
get tester() {
return this.#tester;
}
constructor() {
this.#tester = new CharTester();
this.#transitionTable = [
// State 0
[
new Transition(0,
this.#tester.isBlank,
'nonaccepting',
'notother'),
new Transition(1,
this.#tester.isDigit,
'nonaccepting',
'notother'),
new Transition(2,
this.#tester.isLetter,
'nonaccepting',
'notother'),
new Transition(5,
this.#tester.isParenOpen,
'nonaccepting',
'notother')
],
// State 1
[
new Transition(2,
this.#tester.isLetter,
'nonaccepting',
'notother'),
new Transition(3,
this.#tester.isNotLetterNorDigit,
'number',
'isother')
],
// State 2
[
new Transition(2,
this.#tester.isDigitOrLetter,
'nonaccepting',
'notother'),
new Transition(4,
this.#tester.isNotLetterNorDigit,
'word',
'isother')
],
// State 3: accepting
[],
// State 4: accepting
[],
// State 5
[
new Transition(5,
this.#tester.isNotParenClose,
'nonaccepting',
'isother'),
new Transition(6,
this.#tester.isParenClose,
'comment',
'notother')
],
// State 6: accepting
[]
];
}
}
const Lexer = class {
#transitionTable;
#lineIndex;
#currentState;
constructor() {
this.#transitionTable = new Transitions().table;
this.#lineIndex = 0;
this.#currentState = 0;
}
get lineIndex() {
return this.#lineIndex;
}
get transitionTable() {
return this.#transitionTable;
}
/** @param {Number} state */
set currentState(state) {
this.#currentState = state;
}
debug(line) {
console.log(line[this.#lineIndex]);
}
nextToken(line) {
/* transition has: next state, function, category, isother */
let token = {};
let transitionFound = false;
for (const transition of this.#transitionTable) {
const currentChar = line[this.#lineIndex];
if (transition.callback(currentChar)) {
transitionFound = true;
// if transition is valid: advance state
this.currentState = transition.nextState;
// not valid: evaluate next transition function: ok, forof does it
// nonaccepting state, no valid transition found:
// throw error with offending built token so far
// evaluate until end of string (already happening with forof)
// reset state to 0 if accepting
// nonaccepting, transition found: keep buiding up the token
}
}
// remember to reset the charIndex later
// after lexing the whole current line
}
}
const lexer = new Lexer();
const rl = readline.createInterface({ input, output });
console.log('Enter q to quit');
const loop = async () => {
const answer = await rl.question('VF # ');
if (answer === 'q') {
rl.close();
} else {
lexer.debug(answer);
if (answer !== '') {
console.log('');
}
loop();
}
}
loop();
// TODO
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment