Given string and punctuation, tokenize string on punctuation and whitespace. Done using both split and scan.

tokenize_string.rb

def tokenize_query_split query, punctuation
  punc_regex = /[#{punctuation}\s]+/
  
  tokens = query.split(punc_regex)
  
  tokens.each do |token|
    p token
  end
end

def tokenize_query_scan query, punctuation
  token_regex = /[^#{punctuation}\s]+/
  
  tokens = query.scan(token_regex)
  
  tokens.each do |token|
    p token
  end
end