NLP example

nlp-example.rb

class NLPProcessor

  STOP_WORDS = %w[
    is
    a
    of
    the
  ]

  attr_reader :invert_index

  def process
    remove_stop_words!
    puts @text
    # TODO: remove_endings!
    build_invert_index!
  end

  private

  def initialize(text)
    @text         = text
    @invert_index = Hash.new { |hash, key| hash[key] = [] }
  end

  def remove_stop_words!
    buffer = ''
    
    @text.each_line do |line|
      buffer << line.gsub!(/#{STOP_WORDS.join('|')}/i, '')
    end

    @text = buffer
  end

  def build_invert_index!
    @text.each_line.with_index do |line, index|
      line.split(/\W+/).each do |word|
        @invert_index[word].push(index + 1)
      end
    end
  end
  
end

text = <<~TEXT
It is a briskly blowing wind that blows
from the north, the North of my youth.
The wind is cold too, colder than the
winds of yesteryear.
TEXT

processor = NLPProcessor.new(text)
processor.process
p processor.invert_index