HTTP checker in Go

complex.go

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net"
	"net/http"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"
)

// CustomTransport configures clients
var CustomTransport http.RoundTripper = &http.Transport{
	Proxy: http.ProxyFromEnvironment,
	DialContext: (&net.Dialer{
		Timeout:   30 * time.Second,
		DualStack: true,
	}).DialContext,
	DisableKeepAlives:   true,
	TLSHandshakeTimeout: 10 * time.Second,
}

// CustomClient initiates HTTP requests
var CustomClient = http.Client{
	Transport: CustomTransport,
}

// WorkResult has result data from worker
type WorkResult struct {
	WorkerID int
	URI      string
	Status   int
}

func goodRange(status int) bool {
	return (status >= 200 && status < 400)
}

func ignoreCode(status int) bool {
	return (status == 0 || status == 400 || status == 502)
}

// Status returns a code for WorkResult
func Status(url string) int {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return 0
	}
	req.Header.Add("User-Agent", string(rand.Int()))

	var status = -1
	for i := 0; i < 3; i++ {
		resp, err := CustomClient.Do(req)
		if err != nil {
			re := regexp.MustCompile("(SSL|TLS|x509)")
			if match := re.Find([]byte(err.Error())); match != nil {
				return 0
			}
			<-time.After(1 * time.Second)
		} else {
			resp.Body.Close()
			if goodRange(resp.StatusCode) {
				return resp.StatusCode
			}
			status = resp.StatusCode
			<-time.After(1 * time.Second)
		}
	}

	return status
}

func main() {
	work := make(chan string)

	jsonFile, err := os.Open("links.json")
	if err != nil {
		fmt.Println(err)
		return
	}

	defer jsonFile.Close()

	content, err := ioutil.ReadAll(jsonFile)
	if err != nil {
		fmt.Println(err)
		return
	}

	links := []string{}
	if err := json.Unmarshal(content, &links); err != nil {
		fmt.Println(err)
		return
	}

	go func() {
		ignore := []string{"localhost", "app", "127.0.0.1", "demo"}
		ignorePattern := "(" + strings.Join(ignore, "|") + ")"
		re := regexp.MustCompile(ignorePattern)
		for _, l := range links {
			if match := re.Find([]byte(l)); match != nil {
				continue
			}
			work <- l
		}
		close(work)
	}()

	coeff := math.Log2(float64(len(links)))
	factor := 4.0
	workers := int(coeff * factor)

	wg := &sync.WaitGroup{}
	wg.Add(workers)

	rch := make(chan WorkResult)
	for i := 0; i < workers; i++ {
		go func(ich chan WorkResult, worker int) {
			defer wg.Done()
			for l := range work {
				ich <- WorkResult{
					WorkerID: worker,
					URI:      l,
					Status:   Status(l),
				}
				<-time.After(5 * time.Millisecond)
			}
		}(rch, i)
	}

	go func() {
		wg.Wait()
		close(rch)
	}()

	bad := 0
	good := 0
	total := 0
	for r := range rch {
		fmt.Printf("\rtotal: %4d, failed: %4d, good: %4d", total, bad, good)
		if !goodRange(r.Status) {
			if !ignoreCode(r.Status) {
				fmt.Printf(" -> error %d\n", r.Status)
				bad++
			}
		} else {
			good++
		}
		total++
	}
	fmt.Printf("\rtotal: %4d, failed: %4d, good: %4d\n", total, bad, good)
}

Makefile

all: complex simple
complex: complex.go
	go build complex.go
simple: simple.go
	go build simple.go
clean:
	rm complex simple

simple.go

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strings"
)

// WorkResult has result data from worker
type WorkResult struct {
	WorkerID int
	URI      string
	Status   int
}

func goodRange(status int) bool {
	return (status >= 200 && status < 400)
}

func ignoreCode(status int) bool {
	return (status == 0 || status == 400 || status == 502)
}

// Status returns a code for WorkResult
func Status(url string) int {
	resp, err := http.Get(url)
	if err != nil {
		return -1
	}
	resp.Body.Close()
	return resp.StatusCode
}

func main() {
	jsonFile, err := os.Open("links.json")
	if err != nil {
		fmt.Println(err)
		return
	}

	defer jsonFile.Close()

	content, err := ioutil.ReadAll(jsonFile)
	if err != nil {
		fmt.Println(err)
		return
	}

	links := []string{}
	if err := json.Unmarshal(content, &links); err != nil {
		fmt.Println(err)
		return
	}

	ignore := []string{"localhost", "app", "127.0.0.1", "demo"}
	ignorePattern := "(" + strings.Join(ignore, "|") + ")"
	re := regexp.MustCompile(ignorePattern)

	bad := 0
	good := 0
	total := 0
	for _, l := range links {
		if match := re.Find([]byte(l)); match != nil {
			continue
		}
		fmt.Printf("\rtotal: %4d, failed: %4d, good: %4d", total, bad, good)
		r := WorkResult{1, l, Status(l)}
		if !goodRange(r.Status) {
			if !ignoreCode(r.Status) {
				fmt.Printf(" %v\n", r)
				bad++
			}
		} else {
			good++
		}
		total++
	}
	fmt.Printf("\rtotal: %4d, failed: %4d, good: %4d\n", total, bad, good)
}