Inverse Index

index.cc

std::vector<std::string> InverseIndex::getWords(std::string line)
{
  std::vector<std::string> words;

  char* str = (char*)line.c_str();
  char* end = str + strlen(str) + 1;
  unsigned char symbol[5] = {0,0,0,0,0};

  while( str < end ){
    utf8::uint32_t code = utf8::next(str, end);
    if(code == 0) continue;
    utf8::append(code, symbol);
    // TODO detect whitespaces.
    std::string word = (const char*)symbol;
    words.push_back(word);
  }

  return words;
}