Antelope Lexer Example

lexer.rb

require "strscan"

class Lexer
  def initialize(input)
    @input = input
    @scanner = StringScanner.new(input)
    @line = 1
  end
  
  def lex
    @tokens = []
    until @scanner.eos?
      scan_token or scan_whitespace or error!
    end
    @tokens
  end
  
  def scan_token
    scan_variable or scan_number or scan_operator
  end
  
  def scan_whitespace
    if @scanner.scan(/\s+/)
      @line += @scanner[0].count("\n")
    end
  end
  
  def scan_variable
    if @scanner.scan(/[A-Za-z_][A-Za-z0-9_-]*/)
      emit :ident, @scanner[0]
    end
  end
  
  def scan_number
    if @scanner.scan(/[1-9][0-9]*(\.[0-9]+)?/)
      emit :num, @scanner[0]
    end
  end
  
  def scan_operator
    case
    when @scanner.scan(/\+/)
      emit :plus
    when @scanner.scan(/\-/)
      emit :minus
    when @scanner.scan(/\//)
      emit :divide
    when @scanner.scan(/\*/)
      emit :multiply
    when @scanner.scan(/\=/)
      emit :equals
    else
      nil
    end
  end
  
  private
  
  def error!
    raise SyntaxError, "Unexpected `#{@scanner.check(/./)}' on line #{@line} (column #{column})"
  end
  
  def emit(type, value = nil, line = @line, col = column)
    @tokens << Token.new(type, value, line, col)
  end
  
  def column
    last_newline = @scanner.string.rindex("\n") || 0
    @scanner.pos - last_newline
  end
end

lexer_spec.rb

require_relative "lexer"
require_relative "token"

describe Lexer do
  subject { Lexer.new(input) }
  let(:input) { "test = 3 + 2" }
  
  it "lexes a basic string" do
    expect(subject.lex).to eq [
      token(:ident, "test", 1, 4), 
      token(:equals, nil, 1, 6), 
      token(:num, "3", 1, 8), 
      token(:plus, nil, 1, 10), 
      token(:num, "2", 1, 12)
    ]
  end
  
  def token(*args)
    Token.new(*args)
  end
end

token.rb

# Represents a lexical token.  Our lexical tokens have a line number,
# a column number, a type, and a value.
class Token
  attr_reader :line
  attr_reader :column
  attr_reader :type
  attr_reader :value
  
  def initialize(type, value, line, column)
    @type = type
    @value = value
    @line = line
    @column = column
    freeze
  end
  
  def to_a
    [line, column, type, value].freeze
  end
  
  def ==(other)
    if other.is_a?(Token)
      return to_a == other.to_a
    else
      return to_a == other
    end
  end
end

token_spec.rb

require_relative "token"

describe Token do
  subject { Token.new(:ident, "test", 1, 0) }
  
  it { should be_frozen }
  it { should eq Token.new(:ident, "test", 1, 0) }
end