Handles the slightly odd version of chunked HTTP/1.1 that the S3 SDK uses, but .Net does not handle correctly.

CopyS3ChunkedStream.cs

        /// <summary>
        /// Handles the slightly odd version of chunked HTTP/1.1
        /// that the S3 SDK uses, but .Net does not handle correctly.
        /// </summary>
        private static async Task CopyS3ChunkedStream(Stream input, Stream output)
        {
            // If this is a 'chunk-signature' stream, we expect to see
            // - an ascii string hex number (e.g. 5E24) -- this is the chunk length
            // - the string ";chunk-signature="
            // - a 64-char SHA256 hash check-sum (e.g. "e2617d7cf2c7c91a3b650b16726847b39a6f333577df6617ecd7267e66631df1")
            // - "\r\n"
            // If we don't see that at the start, assume this isn't a chunked stream and just copy everything over

            const int safetyLimit = 16;
            const string chunkSignature = "chunk-signature=";
            const string httpNewLine = "\r\n";
            
            var buffer = new byte[10240];
            var sb = new StringBuilder();
            var bb = new List<byte>();
            
            byte lastByte = 0;
            var stillHaveData = true;

            void Add(int b) { sb.Append((char)b); bb.Add((byte)b); }
            void Clear() { sb?.Clear();bb?.Clear(); }
            async Task<byte> Read() {
                var b = await input.ReadAsync(buffer!, 0, 1);
                if (b < 1) stillHaveData = false;
                lastByte = (byte)b;
                return buffer![0];
            }

            async Task SkipString(string expected)
            {
                for (int i = 0; i < expected.Length; i++) { Add(await Read()); }
                if (!stillHaveData) return;
                var actual = sb.ToString();
                if (actual != expected) throw new Exception($"Unexpected streaming block: Expected '{expected}' but got '{actual}'");
                Clear();
            }

            while (stillHaveData)
            {
                long remainingData = 0;
                
                // Read header
                for (int i = 0; i < safetyLimit; i++)
                {
                    var b = await Read();
                    if (!stillHaveData) break;

                    if (b == ';')
                    {
                        // parse the string and set our expected length
                        var ok = long.TryParse(sb.ToString(), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out remainingData);
                        if (!ok || remainingData < 0) break;
                        if (remainingData == 0) return; // end of stream

                        Clear();
                        break;
                    }

                    Add(b);
                }

                // Check we didn't hit the safety limit
                if (bb.Count > 0) // we didn't see a proper header. Fallback
                {
                    await output.WriteAsync(bb.ToArray(), 0, bb.Count);
                    await input.CopyToAsync(output);
                    return;
                }
                
                // Check for the 'chunk-signature' block
                await SkipString(chunkSignature);
                
                // Read through the hash. Should get a line-break at the end
                for (int i = 0; i < 64; i++) { await Read(); }
                if (!stillHaveData) return;
                await SkipString(httpNewLine);
                
                // Now we should get a chunk of 'real' data
                while (remainingData > 0)
                {
                    var available = (int)Math.Min(buffer.Length, remainingData);
                    
                    var actual = await input.ReadAsync(buffer, 0, available);
                    if (actual < 1) break;
                    remainingData -= actual;
                    await output.WriteAsync(buffer, 0, actual);
                }
                
                // Next should be '\r\n' to start the next chunk header.
                await SkipString(httpNewLine);
            }
        }