Load urls from file and get responses

asyncfetchurls.go

package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

type HttpResponse struct {
	url      string
	response *http.Response
	err      error
}

func getURLsfromFile(filePath string) []string {
	fp, err := os.Open(filePath)
	if err != nil {
		log.Fatal(err)
	}
	defer fp.Close()

	var urls []string
	scanner := bufio.NewScanner(fp)
	for scanner.Scan() {
		if scanner.Text() == "" {
			continue
		}
		urls = append(urls, strings.TrimSpace(scanner.Text()))
	}
	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}
	return urls
}

func asyncHttpGets(urls []string) []*HttpResponse {
	ch := make(chan *HttpResponse, len(urls))
	responses := []*HttpResponse{}
	for _, url := range urls {
		go func(url string) {
			fmt.Printf("Fetching %s \n", url)
			resp, err := http.Get(url)
			ch <- &HttpResponse{url, resp, err}
		}(url)
	}

	for {
		select {
		case r := <-ch:
			fmt.Printf("%s was fetched\n", r.url)
			responses = append(responses, r)
			if len(responses) == len(urls) {
				return responses
			}
		default:
			fmt.Printf(".")
			time.Sleep(5e7)
		}
	}
	return responses
}

func getBodyFromResponses(responses []*HttpResponse) {
	for _, item := range responses {
		body, err := ioutil.ReadAll(item.response.Body)
		if err != nil {
			fmt.Println(err)
		}
		fmt.Println(string(body))
		item.response.Body.Close()
	}
}

func main() {
	urls := getURLsfromFile("urls.txt")
	results := asyncHttpGets(urls)
	//getBodyFromResponses(results)
	for _, item := range results {
		fmt.Println(item.url)
	}
}