Parallel URL Fetcher

main.go

package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"sync"

	"github.com/codegangsta/negroni"
)

const BASE_URL = "http://example.com/"

func main() {
	mux := http.NewServeMux()
	mux.HandleFunc("/example", asyncUrlHandler)

	n := negroni.Classic()
	n.UseHandler(mux)
	n.Run(":3000")
}

func asyncUrlHandler(w http.ResponseWriter, req *http.Request) {
	queryData := req.URL.Query()
	format, ok := queryData["urls"]
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("bad arguments"))
		return
	}
	data := strings.NewReader(format[0])
	r := csv.NewReader(data)
	lines, err := r.Read()
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("bad arguments"))
		return
	}
	retChan := make(chan SiteOut)
	doneChan := make(chan struct{})
	var wg sync.WaitGroup
	for _, v := range lines {
		wg.Add(1)
		go fetchUrl(v, retChan, &wg)
	}
	go func() {
		wg.Wait()
		doneChan <- struct{}{}
	}()
	var output []SiteOut
	for {
		select {
		case v := <-retChan:
			output = append(output, v)
		case <-doneChan:
			json.NewEncoder(w).Encode(output)
			req.Body.Close()
			return
		}
	}
}

func fetchUrl(url string, out chan SiteOut, wg *sync.WaitGroup) {
	defer wg.Done()
	newSite := SiteOut{Base: url}
	fetchUrl := fmt.Sprintf("%s/%s", BASE_URL, url)
	resp, err := http.Get(fetchUrl)
	if err != nil {
		newSite.Data = ""
		out <- newSite
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		newSite.Data = ""
		out <- newSite
		return
	}
	newSite.Data = string(body)
	out <- newSite
}

type SiteOut struct {
	Data string `json:"data"`
	Base string `json:"base"`
}