Test lexer and parser

sexp.rb

class Token
  LPAREN = '('; RPAREN = ')'; EOS = '\00'

  NUM = '1'; STR = '"'; SYM = ':'; EXP = '_'

  NUMSTR = "1234567890."

  def initialize(type, text)
    @type = type; @text = text
  end

  def not_value?
    @type == LPAREN or @type == RPAREN or @type == EOS
  end

  def to_s
    case @type
      when STR then return text.dump
      when SYM then return ":#{text}"
    end
    return @text
  end

  attr_accessor :type, :text
end

def objectize(tok)
  return eval(tok.to_s)
end

def token()
  while true; char = $stdin.getc
    if char == nil then return Token.new(Token::EOS, "EOS") end

    case char
      when '(' then return Token.new(Token::LPAREN, char)
      when ')' then return Token.new(Token::RPAREN, char)

      when '"'
        str = StringIO.new
        while (c = $stdin.getc) != '"'
          str << c
        end
        return Token.new(Token::STR, str.string)
      when ':'
        str = StringIO.new; broken = false
        while (c = $stdin.getc) !~ /[ \n\t]/ and not broken
          if c.match?(/[\(\)]/) then $stdin.ungetc(c); broken = true; break; end
          str << c
        end
        return Token.new(Token::SYM, str.string)
    end

    if Token::NUMSTR.include?(char)
      str = StringIO.new; str << char; broken = false
      while (c = $stdin.getc).match? /[0-9\.\(\)":]/ and not broken
        if c.match?(/[\(\)":]/) then $stdin.ungetc(c); broken = true; break; end
        str << c
      end

      return Token.new(Token::NUM, str.string)
    else
      str = StringIO.new; str << char; broken = false
      while (c = $stdin.getc) !~ /[ \n\t]/ and not broken
        if c == nil then return Token.new(Token::EOS, "EOS") end
        if c.match?(/[\(\)":]/) then $stdin.ungetc(c); broken = true; break; end
        str << c
      end

      return Token.new(Token::EXP, str.string)
    end
  end
end

def parse()
  sexp = []

  while true; if (tok = token()).type == Token::EOS then break end
    warn "=> #{tok.inspect} #{tok}"

    unless tok.not_value?  
      warn "Pushing #{tok}"
      sexp << objectize(tok)
      next
    end

    if token.type == Token::RPAREN
      warn "Leaving paren" 
      return sexp  
    end

    if tok.type == Token::LPAREN
      warn "Entering paren"
      sexp << parse()
    end
  end

  return sexp
end