gistfile1.go

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"
)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func worker(id int, jobs <-chan int, results chan<- int) {
	for j := range jobs {
		//fmt.Println("worker", id, "processing job", j)

		target_url := fmt.Sprintf("http://xxxx.com/yyy/%d", j)
		//fmt.Println(" ", target_url, "\n")
		response, err := http.Get(target_url)
		if err != nil {
			fmt.Printf("%s", err)
			os.Exit(1)
		} else {
			defer response.Body.Close()
			contents, err := ioutil.ReadAll(response.Body)
			if err != nil {
				fmt.Printf("%s", err)
				os.Exit(1)
			}

			file_err := ioutil.WriteFile(fmt.Sprintf("%s%d", "/tmp/go_test/http-", j), contents, 0644)
			check(file_err)
		}

		results <- j

	}
}

func main() {

	workers_size := 1000
	jobs_size := 10000
	chan_size := 10000

	jobs := make(chan int, chan_size)
	results := make(chan int, chan_size)

	now1 := time.Now().UnixNano()

	for w := 1; w <= workers_size; w++ {
		go worker(w, jobs, results)
	}

	for j := 1; j <= jobs_size; j++ {
		jobs <- j
	}

	close(jobs)

	for a := 1; a <= jobs_size; a++ {
		<-results
	}

	now2 := time.Now().UnixNano()

	fmt.Println("--------")
	//fmt.Println(now1)
	//fmt.Println(now2)
	fmt.Println("millisec:", (now2-now1)/1000000)

}