async file downloads

get_irs_migration_data.go

package main

import (
	"fmt"
	"github.com/mitchellh/go-homedir"
	"io"
	"net/http"
	"os"
	"path"
	"sync"
)

var urls = []string{
	"https://www.irs.gov/pub/irs-soi/1990to1991countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/1991to1992countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/1992to1993countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/1993to1994countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/1994to1995countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/1995to1996countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/1996to1997countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/1997to1998countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/1998to1999countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/1999to2000countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/2000to2001countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/2001to2002countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/2002to2003countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/2003to2004countymigration.zip",
	"https://www.irs.gov/pub/irs-soi/county0405.zip",
	"https://www.irs.gov/pub/irs-soi/county0506.zip",
	"https://www.irs.gov/pub/irs-soi/county0607.zip",
	"https://www.irs.gov/pub/irs-soi/county0708.zip",
	"https://www.irs.gov/pub/irs-soi/county0809.zip",
	"https://www.irs.gov/pub/irs-soi/county0910.zip",
	"https://www.irs.gov/pub/irs-soi/county1011.zip",
	"https://www.irs.gov/pub/irs-soi/countyinflow1112.csv",
	"https://www.irs.gov/pub/irs-soi/countyoutflow1112.csv",
	"https://www.irs.gov/pub/irs-soi/countyinflow1213.csv",
	"https://www.irs.gov/pub/irs-soi/countyoutflow1213.csv",
	"https://www.irs.gov/pub/irs-soi/countyinflow1314.csv",
	"https://www.irs.gov/pub/irs-soi/countyoutflow1314.csv",
	"https://www.irs.gov/pub/irs-soi/countyinflow1415.csv",
	"https://www.irs.gov/pub/irs-soi/countyoutflow1415.csv",
}

func downloadFromURL(url string, fileName string) {
	fmt.Println("Downloading", url, "to", fileName)
	output, err := os.Create(fileName)
	if err != nil {
		fmt.Println("Error while creating", fileName, "-", err)
		return
	}
	defer output.Close()

	response, err := http.Get(url)
	if err != nil {
		fmt.Println("Error while downloading", url, "-", err)
		return
	}
	defer response.Body.Close()

	n, err := io.Copy(output, response.Body)
	if err != nil {
		fmt.Println("Error while downloading", url, "-", err)
		return
	}

	fmt.Println(n, "bytes downloaded.")
}

func main() {
	fmt.Println("About to run for", len(urls), "urls")
	var wg sync.WaitGroup
	userHome, err := homedir.Dir()
	if err != nil {
		fmt.Println("Couldn't obtain user home directory -", err)
		return
	}
	rawPath := path.Join(userHome, "data", "irs_migration", "raw")
	if _, err := os.Stat(rawPath); os.IsNotExist(err) {
		os.MkdirAll(rawPath, 0777)
	}

	for _, url := range urls {
		wg.Add(1)
		go func(url string) {
			defer wg.Done()
			fileName := path.Join(rawPath, path.Base(url))
			downloadFromURL(url, fileName)
		}(url)
	}
	wg.Wait()
}