simple tokenizers for ruby

email_tokenizer.rb

class EmailTokenizer < StringTokenizer
  def initialize text
    super text, /(\s|,|;)+/
  end

  def self.filter array=[]
    result = []
    array.each do |item|
      result << item.downcase if item =~ /.+@.+/
    end
    result
  end

  protected
    def tokenize
      self.class.filter super
    end
end

string_tokenizer.rb

class StringTokenizer
  def initialize text='', delim=/\s+/
    raise ArgumentError, 'Text must be a string' if text.nil?
    @text, @delimiter = text, delim
  end

  def tokens
    @tokens ||= tokenize
  end

  protected
    def tokenize
      @text.split @delimiter
    end
end