golang_concurrent_http_request.go

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"sync"
	"time"
)

type result struct {
	index      int
	res        *http.Response
	statusCode int
	err        error
}

var tr *http.Transport

func init() {
	// tr = &http.Transport{MaxIdleConnsPerHost: 10000}
	tr = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext,
		MaxIdleConnsPerHost:   20000,
		MaxIdleConns:          100,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
	}

}

func PostHouse(id int) (*http.Response, error) {
	url := "http://localhost:8099/api/batch"

	client := &http.Client{
		Transport: tr,
		Timeout:   200 * time.Second,
	}

	// client.Timeout = time.Second * 200
	req, err := http.NewRequest("POST", url, nil)
	// req.Header.Set("Content-Type", "application/json")
	// req.Header.Set("Authorization", "Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJzdGF0cy5jb20uYmQiLCJleHAiOjE2MzQ2ODY4MjIsImlhdCI6MTYzNDY2MjgyMiwiaXNzIjoic3RhdHMuY29tLmJkIiwic3ViIjoiNTYyZTliOTYtYTI5My00ZDc2LWE1Y2UtMjM0MzdiYWNjYTUxIiwiUHJtIjoidnFidmJmRTVZdHh0blRMemRyV2tvZFpDTVI3dm1UR1VSa0tZVUdJOVpqVnYzQmxaVUI4ZXdqUEJTNU9iM2YzZiJ9.TXhUQo31iztQke8iQ173aYfPU0kVGjt5SJ_1GWvjj7MFtkveXwMrWMKlI9n0WXcGEiGbt1FQ1hMABfCtiejR15MNHc-zjiA3ODJGRN2p8-KeGdAbfR8jqK144tQ-m_t0SAQ_QUr0vLYdQq89O1Gez7aYbOH-9wu8csRawy4vcCI")

	if err != nil {
		return &http.Response{}, err
	}

	ctx, cancel := context.WithTimeout(req.Context(), 200*time.Second)
	defer cancel()
	req = req.WithContext(ctx)

	res, err := client.Do(req)
	if err != nil {
		return &http.Response{}, err
	}

	// defer res.Body.Close()

	if err != nil {

		return &http.Response{}, err
	}

	// fmt.Println(string(bs))

	return res, nil
}

func main() {
	start := time.Now()
	defer func() {
		fmt.Println("Execution Time: ", time.Since(start).Seconds())
	}()

	concurrencyLimit := 20000

	m := make(map[int]int)
	semaphoreChan := make(chan struct{}, concurrencyLimit)

	resultsChan := make(chan *result, concurrencyLimit)

	wg := sync.WaitGroup{}

	for i := 0; i < concurrencyLimit; i++ {

		wg.Add(1)

		go func(id int) {
			resp, err := PostHouse(id)
			if err != nil {
				fmt.Println("error getting comic", err)
			}

			// fmt.Println(id, statusCode, err)
			semaphoreChan <- struct{}{}

			result := &result{id, resp, resp.StatusCode, err}

			if resp != nil {
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					log.Printf("Error reading body: %v", err)
					// http.Error(w, "can't read body", http.StatusBadRequest)
					// return
				}

				log.Println(string(body))

				resp.Body.Close()
			}

			resultsChan <- result

			<-semaphoreChan

			wg.Done()

		}(i)
	}

	wg.Wait()

	close(semaphoreChan)
	close(resultsChan)
	for v := range resultsChan {
		m[v.statusCode]++
		fmt.Println(v.err)
	}

	for k, v := range m {
		fmt.Println(k, v)
	}
}