JavaのソースファイルをJDT-Coreで字句解析する（必須→'org.eclipse.jdt:org.eclipse.jdt.core:3.10.0'）

JavaTokenizer.java

import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.*;

import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;

import org.eclipse.jdt.internal.compiler.parser.Scanner;

public class JavaTokenizer {
	public static void main(String[] args) throws Exception {
		if (args.length != 2) {
			System.err.println("usage: source.java target.txt");
			return;
		}
		File intfile = new File(args[0]);
		File outfile = new File(args[1]);
		char[] source = null;
		try (FileReader reader = new FileReader(intfile)) {
			StringBuilder sb = new StringBuilder();
			int c;
			while ((c = reader.read()) != -1) {
				sb.append((char) c);
			}
			source = sb.toString().toCharArray();
		} catch (Exception e) {
			e.printStackTrace();
		}
		Scanner scanner = new Scanner();
		scanner.setSource(source);
		scanner.recordLineSeparator = true;
		try (FileWriter writer = new FileWriter(outfile)) {
			while (true) {
				int tokenType = scanner.getNextToken();
				if (tokenType == TokenNameEOF) {
					break;
				}
				int line = scanner.getLineNumber(scanner.getCurrentTokenStartPosition());
				String tokenDesc = String.format("type=%d start=%d end=%d line=%d", tokenType,
						scanner.getCurrentTokenStartPosition(), scanner.getCurrentTokenEndPosition(), line);
				writer.write(scanner.getCurrentTokenString() + " // " + tokenDesc);
				writer.write('\n');
			}
			;
		}
	}
}