RubyLexを触ってみた結果

hello.rb

class Hello
  def hello_world *args
    puts "hello", *args
  end
end

output.out

$ ruby test_rubylex.rb 
#<RubyToken::TkCLASS:0x007ffec402bfc8 @seek=0, @line_no=1, @char_no=0, @name="class">
#<RubyToken::TkSPACE:0x007ffec402b438 @seek=5, @line_no=1, @char_no=5>
#<RubyToken::TkCONSTANT:0x007ffec402a510 @seek=6, @line_no=1, @char_no=6, @name="Hello">
#<RubyToken::TkNL:0x007ffec4029f98 @seek=11, @line_no=1, @char_no=11>
#<RubyToken::TkSPACE:0x007ffec40295e8 @seek=12, @line_no=2, @char_no=0>
#<RubyToken::TkDEF:0x007ffec4028f08 @seek=14, @line_no=2, @char_no=2, @name="def">
#<RubyToken::TkSPACE:0x007ffec4028a80 @seek=17, @line_no=2, @char_no=5>
#<RubyToken::TkIDENTIFIER:0x007ffec402f858 @seek=18, @line_no=2, @char_no=6, @name="hello_world">
#<RubyToken::TkSPACE:0x007ffec402f178 @seek=29, @line_no=2, @char_no=17>
#<RubyToken::TkMULT:0x007ffec402ea48 @seek=30, @line_no=2, @char_no=18, @name="*">
#<RubyToken::TkIDENTIFIER:0x007ffec402dc60 @seek=31, @line_no=2, @char_no=19, @name="args">
#<RubyToken::TkNL:0x007ffec402d1c0 @seek=35, @line_no=2, @char_no=12>
#<RubyToken::TkSPACE:0x007ffec4034178 @seek=36, @line_no=3, @char_no=0>
#<RubyToken::TkIDENTIFIER:0x007ffec4033ae8 @seek=40, @line_no=3, @char_no=4, @name="puts">
#<RubyToken::TkSPACE:0x007ffec40336d8 @seek=44, @line_no=3, @char_no=8>
#<RubyToken::TkSTRING:0x007ffec4032968 @seek=45, @line_no=3, @char_no=9, @value=nil>
#<RubyToken::TkCOMMA:0x007ffec40325a8 @seek=52, @line_no=3, @char_no=16>
#<RubyToken::TkSPACE:0x007ffec4031e78 @seek=53, @line_no=3, @char_no=17>
#<RubyToken::TkMULT:0x007ffec4031680 @seek=54, @line_no=3, @char_no=18, @name="*">
#<RubyToken::TkIDENTIFIER:0x007ffec40387c8 @seek=55, @line_no=3, @char_no=19, @name="args">
#<RubyToken::TkNL:0x007ffec4038408 @seek=59, @line_no=3, @char_no=36>
#<RubyToken::TkSPACE:0x007ffec4037da0 @seek=60, @line_no=4, @char_no=0>
#<RubyToken::TkEND:0x007ffec4037738 @seek=62, @line_no=4, @char_no=2, @name="end">
#<RubyToken::TkNL:0x007ffec40373f0 @seek=65, @line_no=4, @char_no=60>
#<RubyToken::TkEND:0x007ffec4036b80 @seek=66, @line_no=5, @char_no=0, @name="end">
#<RubyToken::TkNL:0x007ffec4036888 @seek=69, @line_no=5, @char_no=66>

test_rubylex.rb

require "irb"

f = open "hello.rb"
r = RubyLex.new
r.set_input f
while t = r.token
  p t
end
f.close