collector.go

package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"
	"strings"
	"os"
)

var controller []string
var url string
var uri string
var port string
var serverlist string

func get(start, done chan bool, prefix string, host string, port string, url string) {
	fmt.Printf("get: prefix:%s host:%s port:%s url:%s\n",prefix,host,port,url)
	if host != "" {
		s := []string{ prefix,host,":",port, url}
		uri := strings.Join(s,"")
		err := errors.New("Error")
		var resp *http.Response
		for {
			<-start
			resp, err = http.Get(uri)
			if err != nil {
				fmt.Printf("XXX %v\n", err)
			} else {
				controller = strings.Split(url,"/")
				f := []string{ host, controller[2] }
				fmt.Printf("file: %s\n", strings.Join(f, "-"))
				fo,err := os.Create(strings.Join(f, "-"))
				if err != nil { panic(err) }
				defer func() {
					if err := fo.Close(); err != nil {
						panic(err)
					}
				}()
				buff,err := ioutil.ReadAll(resp.Body)
				if _, err := fo.Write(buff); err != nil {
					panic(err)
				}
				resp.Body.Close()
			}
			done <- true
		}
	}
}

func main() {
	// c := flag.Int("c", 1, "number of parallel requests")
	port := flag.String("p", "44444", "default pinky port")
	url := flag.String("u", "/pinky/disk", "url to hit")
	serverlist := flag.String("s", "./servers.txt", "List of servers to hit")
	flag.Parse()

	fileBytes, err := ioutil.ReadFile(*serverlist)
	if err != nil {
		panic(err)
	}

	n := 0
	done := make(chan bool, n)
	start := make(chan bool, n)
	b := time.Now().Unix()
	for _, line := range strings.Split(string(fileBytes), "\n") {
		n++
		// Magic here.
	        go get(start, done, "http://", line, *port, *url)
		start <- true
	}
	<-done
	e := time.Now().Unix()
	fmt.Printf("time used %d\n", (e-b))
}

use defer... in goroutine, use panic to handle errors, but do the "done" channel
at the end. If a goroutine paniced, done never gets written, so the app will
block indefinitely. You can defer things like that channel push, since defer
blocks run even on panics.

The goroutine is using shared variables ("controllers"), which will break when
trying to get concurrently... they will overwrite each other.

Don't launch a goroutine, block on receiving from a channel, and then in the parent
push onto a channel to start.. the whole "start" channel is artificial, it doesn't
achieve anything.

Globally defined variables like url, uri, port, etc that are defined globally
but then overwritten locally in functions. THhey don't gain anything and can
create really weird behaviors. Use global variables sparingly.

check errors in most cases, but checking errors on closing a file doesn't
gain much... I never both with it unless it is truely relevant to some type
of cleanup.

don't create an initial error object.. it just creates another object that isn't
necessary, can have "var err error".

It does the get logic in a goroutine with no return/break

It doesn't effectively wait for everything to finish. it receives once on the
done channel, so waits until the first one hits that point, rather than receiving
from the channel n times.

It tries to define a buffered channel with n, but n is zero at the time, so the
channel is actually unbuffered

in the goroutine it checks if host is blank, but doesn't push on the done
channel if it is, so the app can get hung

only define things outside a for loop that need to be... don't need to with
the http response

use time rather than unix... has the benefit of nanosecond precision, and
duration has an easy string method that shows stuff like "37.85ms"