Concurrently Fetch but Sequential Response

main.go

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"sync"
	"time"
)

type Comic struct {
	Num   int    `json:"num"`
	Link  string `json:"link"`
	Img   string `json:"img"`
	Title string `json:"title"`
}

const baseXkcdURL = "https://xkcd.com/%d/info.0.json"

func getComic(comicID int) (comic *Comic, err error) {
	url := fmt.Sprintf(baseXkcdURL, comicID)
	response, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	err = json.NewDecoder(response.Body).Decode(&comic)
	if err != nil {
		return nil, err
	}

	return comic, nil
}

func Test(tasks []int) {
	start := time.Now()
	defer func() {
		fmt.Println("Test : ", time.Since(start))
		return
	}()

	const baseXkcdURL = "https://xkcd.com/%d/info.0.json"
	var requests []http.Request

	for _, i := range tasks {
		req, err := http.NewRequest("GET", fmt.Sprintf(baseXkcdURL, i), nil)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		requests = append(requests, *req)
	}

	_, err := ConcurrentCallsSequentialResponse(requests, func(r *http.Response) (interface{}, error) {
		// fmt.Println("running extractor ", id)
		var comic Comic
		err := json.NewDecoder(r.Body).Decode(&comic)
		if err != nil {
			return nil, err
		}
		// fmt.Println(comic.Title)
		return comic.Title, nil
	})

	if err != nil {
		os.Exit(1)
	}
	// fmt.Println(len(vals))

}

func main() {

	comicsNeeded := []int{}
	for i := 1; i <= 100; i++ {
		comicsNeeded = append(comicsNeeded, i)
	}
	fmt.Println("Load count: ", len(comicsNeeded))

	// fully sequential
	start := time.Now()
	for _, i := range comicsNeeded {
		_, err := getComic(i)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		// fmt.Println(comic.Title)
	}
	fmt.Println("Sequential : ", time.Since(start))

	// concurrency + sorting
	// fmt.Println("Testing...")
	fmt.Println("=========================================")
	Test(comicsNeeded)
	fmt.Println("=========================================")

	// without sort, concurrent
	start = time.Now()
	wg := sync.WaitGroup{}

	defer func() {
		fmt.Println("Concurrent : ", time.Since(start))
	}()

	// comicMap := make(map[int]*Comic, len(comicsNeeded))

	// cache := &sync.Map{}

	// res := make(chan string, len(comicsNeeded))

	for _, id := range comicsNeeded {
		wg.Add(1)
		go func(id int) {
			_, err := getComic(id)
			if err != nil {
				fmt.Println(err)
				os.Exit(1)
			}
			// cache.Store(id, comic.Title)
			// res <- fmt.Sprintf("Fetched comic %d with title %v\n", id, comic.Title)
			// fmt.Println(comic.Title)
			wg.Done()
		}(id)
	}
	wg.Wait()
	// close(res)

	// var keys = []int{}

	// cache.Range(func(key, value interface{}) bool {
	// 	keys = append(keys, key.(int))
	// 	return true
	// })

	// sort.Ints(keys)

}

utils.go

package main

import (
	"net/http"
	"os"
	"sort"
	"sync"
)

func ConcurrentCallsSequentialResponse(requests []http.Request, extractor func(*http.Response) (interface{}, error)) (results []interface{}, err error) {

	cache := &sync.Map{}
	wg := sync.WaitGroup{}

	for n, r := range requests {
		wg.Add(1)
		go func(n int, r http.Request) {
			defer wg.Done()
			// fmt.Println("running request ", n)
			res, err := http.DefaultClient.Do(&r)
			if err != nil {
				os.Exit(1)
			}
			// Do something with the response
			result, err := extractor(res)
			if err != nil {
				os.Exit(1)
			}
			cache.Store(n, result)
		}(n, r)
	}
	wg.Wait()

	keys := []int{}

	cache.Range(func(key, value interface{}) bool {
		keys = append(keys, key.(int))
		return true
	})

	sort.Ints(keys)
	for _, k := range keys {
		if v, ok := cache.Load(k); ok {
			results = append(results, v)
		}
	}

	return results, nil
}

Load count:  100
Sequential :  2.818777099s
=========================================
Test :  454.217906ms
=========================================
Concurrent :  314.601457ms