Hand written CSV tokenizer

tokenizer.rb

class Tokenizer

  class Buffer # :nodoc:
    def initialize io
      @io     = io
      @buffer = []
    end

    def peek len = 1
      (len - @buffer.length).times {
        @buffer << @io.getc
      }
      @buffer.first(len).join
    end

    def getc
      if @buffer.empty?
        @io.getc
      else
        @buffer.shift
      end
    end

    def read len
      len.times.map { getc }.join
    end

    def ungetc c
      @buffer << c
    end
  end

  def initialize(io)
    @buf = Buffer.new io
  end

  def next_token
    c = @buf.getc
    case c
    when '"' then [:VALUE, scan_quoted_value]
    when /[^,\r\n]/
      @buf.ungetc(c)
      [:VALUE, scan_value]
    else
      [c,c]
    end
  end

  def scan_value
    val = ''

    while @buf.peek =~ /[^,\n\r]/
      val << @buf.getc
    end

    "#{val}"
  end

  def scan_quoted_value
    val = ''
    loop do
      c = @buf.getc
      case c
      when /[^"]/ then val << c
      when '"'
        case @buf.peek
        when '"'
          val << @buf.getc
        else
          break
        end
      else
        raise "wtf"
      end
    end

    "\"#{val}\""
  end

end