snippet.rb

require 'yard'

class IncrementalArray
  def initialize(block) 
    @block = block
    @buffer = YARD::Parser::Ruby::Legacy::TokenList.new
  end
  def shift; fill_buffer; @buffer.shift end
  def first; fill_buffer; @buffer.first end
  def [](index) fill_buffer; @buffer[index] end
  
  private
  
  def fill_buffer
    @buffer.push(@block.call) if @buffer.empty?
  end
end

class IncrementalParser < YARD::Parser::Ruby::Legacy::StatementList
  def initialize(&block) @tokens = IncrementalArray.new(block) end
  public :next_statement
end

@tokens = ["def foo; end; xxx", "line 2", "line 3", "..."]
parser = IncrementalParser.new { @tokens.shift }
p parser.next_statement