call-url-concurrent.go

package main

import (
	"bufio"
	"fmt"
	"io"
	"net/http"
	"os"
	"sync"
)

func readUrlsFrom(source *os.File) []string {
	var urls []string

	r := bufio.NewReader(os.Stdin)
	for {
		line, _, err := r.ReadLine()
		url := string(line)

		if url != "" {
			urls = append(urls, url)
		}

		if err == io.EOF {
			break
		}
	}
	return urls
}

func main() {
	urls := readUrlsFrom(os.Stdin)
	channel := make(chan string)
	var wg sync.WaitGroup

	wg.Add(len(urls))

	for _, url := range urls {
		go callURL(url, channel, &wg)
	}

	go func() {
		for status := range channel {
			fmt.Println(status)
		}
	}()

	wg.Wait()
}

func callURL(url string, channel chan string, wg *sync.WaitGroup) {
	defer wg.Done()

	resp, err := http.Get(url)
	if err != nil {
		channel <- fmt.Sprintf("%s: %s", url, err.Error())
		return
	}

	channel <- fmt.Sprintf("%s: %s", url, resp.Status)
}

call-url-synchronous.go

package main

import (
	"bufio"
	"fmt"
	"io"
	"net/http"
	"os"
)

func readUrlsFrom(source *os.File) []string {
	var urls []string

	r := bufio.NewReader(os.Stdin)
	for {
		line, _, err := r.ReadLine()
		url := string(line)

		if url != "" {
			urls = append(urls, url)
		}

		if err == io.EOF {
			break
		}
	}
	return urls
}

func main() {
	urls := readUrlsFrom(os.Stdin)

	for _, url := range urls {
		fmt.Println(callURL(url))
	}
}

func callURL(url string) string {
	resp, err := http.Get(url)
	if err != nil {
		return fmt.Sprintf("%s: %s", url, err.Error())
	}

	return fmt.Sprintf("%s: %s", url, resp.Status)
}

try-out.md

$ time go run call-url-concurrent.go < urls.txt >/dev/null

real	0m1.523s
user	0m0.474s
sys	0m0.109s

---

$ time go run call-url-synchronous.go < urls.txt >/dev/null

real	0m5.349s
user	0m0.444s
sys	0m0.120s

urls.txt

http://www.google.com
http://www.github.com
http://www.facebook.com
http://www.youtube.com
http://www.baidu.com
http://www.yahoo.com
http://www.wikipedia.com
http://www.amazon.com
http://www.twitter.com
http://www.linkedin.com