Count the number of "Go" word in the list of URLs.

README.md

• Take a list of URLs from the os.Stdin.
• For every URL in the list start a goroutines.
• Do not run more than k=5 goroutines.
• Do not run more than len(URLs) goroutines (e.g. if k=1000 and len(urls)=10).

main.go

package main

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"sync"
)

/*
$ echo -e 'https://golang.org\nhttps://golang.org\nhttps://golang.org' | go run main.go
Count for https://golang.org: 9
Count for https://golang.org: 9
Count for https://golang.org: 9
Total: 27
*/

const (
	findStr        = "Go"
	maxConcurrency = 5
)

func main() {
	var (
		total int
		wg    sync.WaitGroup
	)

	sc := bufio.NewScanner(os.Stdin)

	limiter := make(chan struct{}, maxConcurrency)
	results := make(chan int)

	for sc.Scan() {
		limiter <- struct{}{}
		wg.Add(1)

		u := sc.Text()

		go func(u string) {
			defer func() { <-limiter }()
			n := CountInURL(u)
			fmt.Printf("Count for %s: %d\n", u, n)
			results <- n
		}(u)
	}

	go func() {
		for n := range results {
			total += n
			wg.Done()
		}
	}()

	wg.Wait()

	fmt.Printf("Total: %d\n", total)
}

func CountInURL(u string) int {
	body, err := getURL(u)
	if err != nil {
		fmt.Printf("error for %s: %v\n", u, err)
		return 0
	}
	return strings.Count(string(body), findStr)
}

func getURL(u string) ([]byte, error) {
	if u = strings.TrimSpace(u); u == "" {
		return nil, errors.New("empty URL")
	}

	resp, err := http.Get(u)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("bad response status: %s", resp.StatusCode)
	}
	return ioutil.ReadAll(resp.Body)
}