Attempt to understand how Golang's `net/http` package handles compressed responses

httpgzip.go

package main

import (
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
)

type Experiment struct {
	URL          string
	ExtraHeaders map[string]string
}

func (e Experiment) Describe() string {
	s := e.URL
	for k, v := range e.ExtraHeaders {
		s += fmt.Sprintf(" %s=%s", k, v)
	}
	return s
}

func main() {
	addAcceptEncodingHeader := map[string]string{
		"Accept-Encoding": "gzip",
	}

	experiments := []Experiment{
		{"http://httpbin.org/get", nil},
		{"http://httpbin.org/gzip", nil},
		{"http://httpbin.org/get", addAcceptEncodingHeader},
		{"http://httpbin.org/gzip", addAcceptEncodingHeader},
	}

	for _, e := range experiments {
		fmt.Printf("==== %s ====\n", e.Describe())

		req, err := http.NewRequest(http.MethodGet, e.URL, nil)
		if err != nil {
			log.Fatal(err)
		}

		for k, v := range e.ExtraHeaders {
			req.Header.Add(k, v)
		}

		err = DebugRequest(req)
		if err != nil {
			log.Fatal(err)
		}

		fmt.Println("")
	}
}

func DebugRequest(req *http.Request) error {
	b, err := httputil.DumpRequestOut(req, false)
	if err != nil {
		return err
	}

	fmt.Println(string(b))

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	b, err = httputil.DumpResponse(res, false)
	if err != nil {
		return err
	}

	fmt.Println(string(b))

	fmt.Println("Uncompressed", res.Uncompressed)

	return nil
}

notes.md

Summary is that providing the Accept-Encoding header yourself disables automatic decompression.

output.txt

==== http://httpbin.org/get ====
GET /get HTTP/1.1
Host: httpbin.org
User-Agent: Go-http-client/1.1
Accept-Encoding: gzip


HTTP/1.1 200 OK
Content-Length: 204
Access-Control-Allow-Credentials: true
Access-Control-Allow-Origin: *
Connection: keep-alive
Content-Type: application/json
Date: Mon, 30 Jan 2017 06:30:55 GMT
Server: nginx


Uncompressed false

==== http://httpbin.org/gzip ====
GET /gzip HTTP/1.1
Host: httpbin.org
User-Agent: Go-http-client/1.1
Accept-Encoding: gzip


HTTP/1.1 200 OK
Access-Control-Allow-Credentials: true
Access-Control-Allow-Origin: *
Connection: keep-alive
Content-Type: application/json
Date: Mon, 30 Jan 2017 06:30:55 GMT
Server: nginx


Uncompressed true

==== http://httpbin.org/get Accept-Encoding=gzip ====
GET /get HTTP/1.1
Host: httpbin.org
User-Agent: Go-http-client/1.1
Accept-Encoding: gzip


HTTP/1.1 200 OK
Content-Length: 204
Access-Control-Allow-Credentials: true
Access-Control-Allow-Origin: *
Connection: keep-alive
Content-Type: application/json
Date: Mon, 30 Jan 2017 06:30:55 GMT
Server: nginx


Uncompressed false

==== http://httpbin.org/gzip Accept-Encoding=gzip ====
GET /gzip HTTP/1.1
Host: httpbin.org
User-Agent: Go-http-client/1.1
Accept-Encoding: gzip


HTTP/1.1 200 OK
Content-Length: 163
Access-Control-Allow-Credentials: true
Access-Control-Allow-Origin: *
Connection: keep-alive
Content-Encoding: gzip
Content-Type: application/json
Date: Mon, 30 Jan 2017 06:30:55 GMT
Server: nginx


Uncompressed false