lex.d

import std.algorithm;
import std.exception;
import std.stdio;
import std.range;
import std.traits;
import std.typecons;

class InvalidCharacerException : Exception {
    this(string msg, string file = __FILE__, uint line = __LINE__) {
        super(msg, file, line);
    }
}

auto lexChar(alias Char, Range)(Range range) if (isInputRange!Range) {
    return range.until!(a => a != Char)();
}

auto lexPlus(Range)(Range range) {
    return lexChar!('+', Range)(range);
}
auto lexMinus(Range)(Range range) {
    return lexChar!('-', Range)(range);
}
auto lexStar(Range)(Range range) {
    return lexChar!('*', Range)(range);
}
auto lexSlash(Range)(Range range) {
    return lexChar!('/', Range)(range);
}
auto lexLParen(Range)(Range range) {
    return lexChar!('(', Range)(range);
}
auto lexRParen(Range)(Range range) {
    return lexChar!(')', Range)(range);
}

auto lexNumeric(Range)(Range range) if (isInputRange!Range) {
    return range.until!(a => !std.uni.isNumber(a))();
}

struct Tokenizer(Args...) {
    alias Lex = Args;
    alias TokenType = typeof(Args[1]);
}

struct Token(Type, Range) {
    Type type;
    Range value;
}

struct Tokenize(Tokenizer, Range) if (isForwardRange!Range) {
    alias Token = .Token!(Tokenizer.TokenType, ElementType!Range[]);
    private Range _range;
    private Token _front;
    private bool _empty;

    this(Range range) {
        _range = range.save();

        _empty = _range.empty();
        if (!_empty) {
            popFront();
        }
    }

    auto front() const {
        return _front;
    }

    auto popFront() in {
        assert(!empty);
    } body {
        if (_range.empty) {
            _empty = true;
        } else {
            foreach (i, lex; Tokenizer.Lex) {
                static if (i % 2 == 0) {
                    auto lexed = lex(_range);
                    if (!lexed.empty) {
                        _front = Token(Tokenizer.Lex[i + 1], lexed.array);
                        _range.popFrontN(_front.value.length);
                        return;
                    }
                }
            }
            throw new InvalidCharacerException(_range);
        }
    }

    auto empty() const {
        return _empty;
    }

    Tokenize!(Tokenizer, Range) save() {
        return Tokenize!(Tokenizer, Range)(_range);
    }
}
auto tokenize(Tokenizer, Range)(Range range) {
    return Tokenize!(Tokenizer, Range)(range);
}

enum TokenType {
    Numeric, Plus, Minus, Star, Slash, LParen, RParen,
}

unittest {
    auto range = "12+20*3-4*(22+3)";

    alias tokenizer = Tokenizer!(
            lexNumeric, TokenType.Numeric,
            lexPlus, TokenType.Plus,
            lexMinus, TokenType.Minus,
            lexStar, TokenType.Star,
            lexSlash, TokenType.Slash,
            lexLParen, TokenType.LParen,
            lexRParen, TokenType.RParen);

    auto tokens = tokenize!(tokenizer)(range);

    static assert(isInputRange!(typeof(tokens)));
    //static assert(isForwardRange!(typeof(tokens)));

    writeln(tokens);
    assert(equal(tokens, [
                Token!(TokenType, dchar[])(TokenType.Numeric, "12"d.dup),
                Token!(TokenType, dchar[])(TokenType.Plus, "+"d.dup),
                Token!(TokenType, dchar[])(TokenType.Numeric, "20"d.dup),
                Token!(TokenType, dchar[])(TokenType.Star, "*"d.dup),
                Token!(TokenType, dchar[])(TokenType.Numeric, "3"d.dup),
                Token!(TokenType, dchar[])(TokenType.Minus, "-"d.dup),
                Token!(TokenType, dchar[])(TokenType.Numeric, "4"d.dup),
                Token!(TokenType, dchar[])(TokenType.Star, "*"d.dup),
                Token!(TokenType, dchar[])(TokenType.LParen, "("d.dup),
                Token!(TokenType, dchar[])(TokenType.Numeric, "22"d.dup),
                Token!(TokenType, dchar[])(TokenType.Plus, "+"d.dup),
                Token!(TokenType, dchar[])(TokenType.Numeric, "3"d.dup),
                Token!(TokenType, dchar[])(TokenType.RParen, ")"d.dup),
                ]));
}