Skip to content

Instantly share code, notes, and snippets.

@takkkun
Last active December 17, 2015 18:49
Show Gist options
  • Select an option

  • Save takkkun/5655587 to your computer and use it in GitHub Desktop.

Select an option

Save takkkun/5655587 to your computer and use it in GitHub Desktop.
import org.atilika.kuromoji._
import collection.JavaConversions._
class Word(surface: String) {
override val toString = surface
}
object Word {
def fromToken(token: Token): Word = {
val surface = token.getSurfaceForm
token.getAllFeatures.split(',').headOption match {
case Some("名詞") => Noun(surface)
case Some("形容詞") => Adjective(surface)
case Some("助詞") => Particle(surface)
case _ => Other(surface)
}
}
}
case class Noun (surface: String) extends Word(surface)
case class Adjective(surface: String) extends Word(surface)
case class Particle (surface: String) extends Word(surface)
case class Other (surface: String) extends Word(surface)
class Suddenizer {
val SUDDEN_DEATH_STRING = """
_人人人人人人_
> 突然の死 <
 ̄YYYYYY ̄
"""
lazy val tokenizer = Tokenizer.builder().build()
type BoundaryPattern = PartialFunction[Seq[Word], Option[Word]]
var canDecoupledAt: BoundaryPattern = { case _ => None }
def <<(pattern: BoundaryPattern) {
canDecoupledAt = pattern.orElse(canDecoupledAt)
}
def apply(text: String): Option[String] =
takeWords(tokenizer.tokenize(text).map(Word.fromToken)) match {
case Nil => None
case words => Some(words.map(_.toString).reduceLeft(_ + _) + SUDDEN_DEATH_STRING)
}
def takeWords(words: Seq[Word]): Seq[Word] =
findBoundaries(words).reverse match {
case lastBoundary :: _ => words.reverse.dropWhile(lastBoundary !=).reverse
case _ => Seq.empty
}
def findBoundaries(words: Seq[Word]): Seq[Word] = findBoundaries(words, List.empty)
def findBoundaries(words: Seq[Word], boundaries: List[Word]): Seq[Word] =
words match {
case Seq(_, rest @ _*) => findBoundaries(rest, canDecoupledAt(words).toList ::: boundaries)
case _ => boundaries.reverse
}
}
object SuddenKiller {
def main(args: Array[String]) {
val input = args.firstOption.getOrElse("")
val suddenizer = new Suddenizer
suddenizer << {
case Seq(Noun(_), boundary @ Particle("の"), _*) => Some(boundary)
case Seq(Noun(_), boundary @ Particle("が"), _*) => Some(boundary)
}
suddenizer << {
case Seq(boundary @ Adjective(_), _*) => Some(boundary)
}
val suddenized = suddenizer(input).getOrElse(input)
println(suddenized)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment