I got bored and implemented a base64 encoder/decoder. This is much slower than the .NET implementation.

Base64Extensions.cs

    public static class Base64Extensions
    {
        private static readonly Lazy<char[]> EncMap =
            new Lazy<char[]>(BuildEncodingMap);

        private static readonly Lazy<int[]> DecMap =
            new Lazy<int[]>(BuildDecodingMap);

        private const char Pad = '=';
        private const int InvChar = -1;

        public static IEnumerable<char> ToBase64(this IEnumerable<byte> bytes)
        {
            if (bytes == null)
                throw new ArgumentNullException("bytes");

            var it = bytes.GetEnumerator();

            while (it.MoveNext())
            {
                var bits = it.Current << 8;

                if (it.MoveNext())
                {
                    bits = (bits | it.Current) << 8;
                }
                else
                {
                    // have 16: 111111 110000 0000
                    yield return Encode(bits >> 10);
                    yield return Encode((bits >> 4) & 63);

                    yield return Pad;
                    yield return Pad;
                    yield break;
                }

                if (it.MoveNext())
                {
                    bits = bits | it.Current;
                }
                else
                {
                    // have 24: 111111 111111 111100 000000
                    yield return Encode(bits >> 18);
                    yield return Encode((bits >> 12) & 63);
                    yield return Encode((bits >> 6) & 63);

                    yield return Pad;
                    yield break;
                }

                // have 24: 111111 111111 111111 111111
                yield return Encode(bits >> 18);
                yield return Encode((bits >> 12) & 63);
                yield return Encode((bits >> 6) & 63);
                yield return Encode(bits & 63);
            }
        }

        public static IEnumerable<byte> FromBase64(this IEnumerable<char> base64Chars)
        {
            if (base64Chars == null)
                throw new ArgumentNullException("base64Chars");

            var it = base64Chars.GetEnumerator();
            int n;

            while (Decode(it, out n))
            {
                var bits = n << 6;

                if (Decode(it, out n))
                {
                    bits = (bits | n) << 6;
                }
                else
                {
                    // have 12: 11111111 0000
                    yield return (byte)(bits >> 4);
                    yield break;
                }

                if (Decode(it, out n))
                {
                    bits = (bits | n) << 6;
                }
                else
                {
                    // have 18: 11111111 00000000 00
                    yield return (byte)(bits >> 10);
                    yield break;
                }

                if (Decode(it, out n))
                {
                    bits = bits | n;
                }
                else
                {
                    // have 24: 11111111 11111111 00000000
                    yield return (byte)(bits >> 16);
                    yield return (byte)((bits >> 8) & 255);
                    yield break;
                }

                // have 24: 11111111 11111111 11111111
                yield return (byte)(bits >> 16);
                yield return (byte)((bits >> 8) & 255);
                yield return (byte)(bits & 255);
            }
        }

        private static char Encode(int n)
        {
            return EncMap.Value[n];
        }

        private static bool Decode(IEnumerator<char> it, out int n)
        {
            while (true)
            {
                if (!it.MoveNext())
                {
                    n = InvChar;
                    return false;
                }

                // ignore whitespace
                if (!char.IsWhiteSpace(it.Current))
                    break;
            }

            var decMap = DecMap.Value;

            var c = it.Current;
            if (c == Pad)
            {
                n = InvChar;
                return false;
            }

            const string invalidCharMsg = "Encountered an invalid base64 character.";

            try
            {
                n = decMap[c];
            }
            catch (IndexOutOfRangeException)
            {
                throw new FormatException(invalidCharMsg);
            }

            if (n == InvChar)
                throw new FormatException(invalidCharMsg);

            return true;
        }

        private static char[] BuildEncodingMap()
        {
            return Enumerable.Range('A', 26)
                .Concat(Enumerable.Range('a', 26))
                .Concat(Enumerable.Range('0', 10))
                .Concat(Enumerable.Range('+', 1))
                .Concat(Enumerable.Range('/', 1))
                .Select(x => (char)x)
                .ToArray();
        }

        private static int[] BuildDecodingMap()
        {
            var encMap = EncMap.Value;
            var decMap = new int[encMap.Max() + 1];

            for (var i = 0; i < decMap.Length; i++)
                decMap[i] = InvChar;

            for (var i = 0; i < encMap.Length; i++)
                decMap[encMap[i]] = i;

            return decMap;
        }
    }