Skip to content

Instantly share code, notes, and snippets.

@king1600
Created July 3, 2018 21:24
Show Gist options
  • Save king1600/18efe381cd2b684566b26fa71b19c92b to your computer and use it in GitHub Desktop.
Save king1600/18efe381cd2b684566b26fa71b19c92b to your computer and use it in GitHub Desktop.
(Language) Lexer Generator written in pony
use "regex"
class Rule[T]
let name: String
let regex: String
let _create: {(String): T^}
new create(name': String, regex': String, make: {(String): T^}) =>
name = name'
regex = regex'
_create = make
fun from(text: String): T^ =>
_create(consume text)
class Lexer[T]
let _regex: Regex
let rules: Array[Rule[T]]
new create(rules': Array[Rule[T]]) ? =>
let rule_list = Array[String](rules'.size())
for rule in rules'.values() do
rule_list.push(
(recover String(rule.name.size()) end)
.>append("(?<").>append(rule.name).>push('>')
.>append(rule.regex).>push(')'))
end
_regex = Regex("|".join(rule_list.values()))?
rules = consume rules'
fun lex(input: String): LexemeIter[T]^ =>
LexemeIter[T](this, _regex.matches(consume input))
class LexemeIter[T] is Iterator[T]
let _lexer: Lexer[T] box
let _iter: MatchIterator
new create(lexer: Lexer[T] box, iter: MatchIterator) =>
_iter = iter
_lexer = lexer
fun ref has_next(): Bool =>
_iter.has_next()
fun ref next(): T^ ? =>
let m = _iter.next()?
for rule in _lexer.rules.values() do
try return rule.from(m.find(rule.name)?) end
end
error
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment