lucene の analyzer を使って tokenize だけしてみるデモ(3.x用)

LuceneTokenizeDemo.java

package com.arielnetworks.agn;

import java.io.StringReader;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cjk.CJKAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;

public class LuceneTokenizeDemo {
    public static void main(String[] args) throws Exception {
        try (Analyzer a = new CJKAnalyzer(Version.LUCENE_36)) {
            TokenStream ts = a.tokenStream("body", new StringReader("システム第１事業部"));
            while (ts.incrementToken()) {
                CharTermAttribute cta = ts.getAttribute(CharTermAttribute.class);
                System.out.println(cta);
            }
        }
    }
}