gistfile1.txt

public partial class Tokenizer
    {
        static Tokenizer()
        {
            var space = Chars.OneOf('\u0020', '\u0009', '\u000a', '\u000d');

            var newline = Combinator.Choice(
                Chars.Sequence("\r\n"),
                Chars.Sequence("\r"),
                Chars.Sequence("\n"));

            var lineComment = Chars.Sequence("//").Right(newline.Not().Right(Chars.Any()).Many()).Ignore();

            Parser<Char, Unit> blockCommentRef = null;

            var blockCommentBegin = Chars.Sequence("/*");
            var blockCommentEnd = Chars.Sequence("*/");

            var blockComment = new Lazy<Parser<Char, Unit>>(() => blockCommentRef);

            var blockCommentRest = blockCommentBegin.Not().Right(blockCommentEnd.Not()).Right(Chars.Any().Ignore())
                .Or(() => blockComment.Value);

            blockCommentRef = blockCommentBegin.Right(blockCommentRest.Many().Ignore()).Left(blockCommentEnd);

            var @int = Chars.Digit().Many();

            var @int16 = @int.Left(Chars.OneOf('s', 'S'))
                .Select(_ => new Token(TokenKind.Int16, Convert.ToInt16(new String(_.ToArray()))));
            var @int32 = @int
                .Select(_ => new Token(TokenKind.Int32, Convert.ToInt32(new String(_.ToArray()))));
            var @int64 = @int.Left(Chars.OneOf('l', 'L'))
                .Select(_ => new Token(TokenKind.Int64, Convert.ToInt64(new String(_.ToArray()))));

            var oct = Chars.Sequence("0o").Right(Chars.Oct().Many());

            var oct16 = oct.Left(Chars.OneOf('s', 'S'))
                .Select(_ => new Token(TokenKind.Int16, Convert.ToInt16(new String(_.ToArray()), 8)));
            var oct32 = oct
                .Select(_ => new Token(TokenKind.Int32, Convert.ToInt32(new String(_.ToArray()), 8)));
            var oct64 = oct.Left(Chars.OneOf('l', 'L'))
                .Select(_ => new Token(TokenKind.Int64, Convert.ToInt64(new String(_.ToArray()), 8)));

            var hex = Chars.Sequence("0x").Right(Chars.Hex().Many());

            var hex16 = hex.Left(Chars.OneOf('s', 'S'))
                .Select(_ => new Token(TokenKind.Int16, Convert.ToInt16(new String(_.ToArray()), 16)));
            var hex32 = hex
                .Select(_ => new Token(TokenKind.Int32, Convert.ToInt32(new String(_.ToArray()), 16)));
            var hex64 = hex.Left(Chars.OneOf('l', 'L'))
                .Select(_ => new Token(TokenKind.Int64, Convert.ToInt64(new String(_.ToArray()), 16)));

            var @float = @int
                .Pipe('.'.Satisfy().Pipe(@int, (x, y) => x.Concat(y)),
                    Chars.OneOf('e', 'E')
                        .Pipe(Chars.OneOf('+', '-')
                            .Pipe(@int, (x, y) => x.Concat(y))
                                .Or(@int), (x, y) => x.Concat(y))
                        .Maybe().Select(_ => _.Otherwise(() => Enumerable.Empty<Char>())),
                    (x, y, z) => x.Concat(y).Concat(z));

            var @single = @float.Left(Chars.OneOf('f', 'F'))
                .Select(_ => new Token(TokenKind.Single, Convert.ToSingle(new String(_.ToArray()))));

            var @double = @float
                .Select(_ => new Token(TokenKind.Double, Convert.ToDouble(new String(_.ToArray()))));

            var letter = (Chars.Eof().Not()).Right(Chars.NoneOf('\u000a', '\u000d'));

            var @char = letter.Between('\''.Satisfy(), '\''.Satisfy())
                .Select(_ => new Token(TokenKind.Char, _));

            var @string = letter.Many().Between('\"'.Satisfy(), '\"'.Satisfy())
                .Select(_ => new Token(TokenKind.String, new String(_.ToArray())));

            var lParen = '('.Satisfy().Select(_ => new Token(TokenKind.LParen, _));
            var rParen = ')'.Satisfy().Select(_ => new Token(TokenKind.LParen, _));

            var lBrace = '{'.Satisfy().Select(_ => new Token(TokenKind.LBrace, _));
            var rBrace = '}'.Satisfy().Select(_ => new Token(TokenKind.RBrace, _));

            var lBracket = '['.Satisfy().Select(_ => new Token(TokenKind.LBracket, _));
            var rBracket = ']'.Satisfy().Select(_ => new Token(TokenKind.RBracket, _));


            var punctuation = Combinator.Choice(
                ','.Satisfy().Select(_ => new Token(TokenKind.Comma, _)),
                '.'.Satisfy().Select(_ => new Token(TokenKind.Period, _)),
                ':'.Satisfy().Select(_ => new Token(TokenKind.Colon, _)),
                ';'.Satisfy().Select(_ => new Token(TokenKind.SemiColon, _)));

            var skip = Combinator.Choice(space.Ignore(), lineComment, blockComment.Value).Many().Ignore();

            var token = skip.Right(Combinator.Choice(
                    lParen,
                    rParen,
                    lBrace,
                    rBrace,
                    lBracket,
                    rBracket,
                    punctuation,
                    @int16,
                    @int32,
                    @int64,
                    @single,
                    @double,
                    oct16,
                    oct32,
                    oct64,
                    hex16,
                    hex32,
                    hex64,
                    @char,
                    @string
                    )).Many().Left(Errors.FollowedBy(Chars.Eof(), "Unexpected Eof"));

        }

    }