Simple KyTea word segmenter-only pipeline.

kytea_word_segmenter.cc

// Wrapper class of KyTea word segmenter.
// Author: odashi
// Date: 2021-01-26
// License: MIT

#include <memory>
#include <string>

#include "kytea/kytea.h"
#include "kytea/string-util.h"

using kytea::Kytea;
using kytea::KyteaConfig;
using kytea::KyteaSentence;

class WordSegmenter {
 public:

  // ctor. Currently this takes only a model path.
  WordSegmenter(std::string const& model_path) {
    auto config = std::make_unique<KyteaConfig>();
    config->setEncoding("utf8");
    config->setDoTags(false);

    kytea_ = std::make_unique<Kytea>(config.release());
    kytea_->readModel(model_path.c_str());
  }

  // Performs segmentation.
  std::vector<std::string> Perform(std::string const& input) {
    auto* str_util = kytea_->getStringUtil();

    auto sentence = std::make_unique<KyteaSentence>();
    sentence->surface = str_util->mapString(input);
    sentence->norm = str_util->normalize(sentence->surface);
    if (sentence->surface.length() != 0) {
      sentence->wsConfs.resize(sentence->surface.length() - 1, 0.0);
    }

    kytea_->calculateWS(*sentence);

    std::vector<std::string> words;
    words.reserve(sentence->words.size());
    for (auto const& word : sentence->words) {
      words.push_back(str_util->showString(word.surface));
    }

    return words;
  }

 private:
  std::unique_ptr<Kytea> kytea_;
};