atmega32u4が品切れだった時に電子部品サイトの在庫をscrapeするくん

main.go

package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
)

type stockExtractor func(in string) (stock int64)
type scraper func(partno string, stock chan<- int64)

func sumNumberInURL(url, selector string, extractor stockExtractor) int64 {
	// fmt.Println("Fetching: ", url)

	var sum int64
	doc, err := goquery.NewDocument(url)
	if err != nil {
		fmt.Printf("failed to GET: \"%s\"\n%v\n", url, err)
		os.Exit(1)
	}

	selection := doc.Find(selector)
	if selection.Length() == 0 {
		fmt.Printf("URL: \"%s\" doesn't include \"%s\"\n", url, selector)
		os.Exit(1)
	}

	selection.Each(func(_ int, s *goquery.Selection) {
		val := s.Text()
		// fmt.Println("val: ", val)

		vali := extractor(val)
		// fmt.Println("vali: ", vali)

		sum += vali
	})

	// fmt.Println(url, sum)
	return sum
}

func ScrapeDigikey(partno string, stock chan<- int64) {
	// ex: http://www.digikey.jp/product-search/ja?KeyWords=atmega32u4-au
	Url, _ := url.Parse("http://www.digikey.jp/product-search/ja")
	parameters := url.Values{}
	parameters.Add("KeyWords", partno)
	Url.RawQuery = parameters.Encode()
	// fmt.Println("Encoded URL is ", Url.String())

	sum := sumNumberInURL(
		Url.String(),
		"td.qtyAvailable",
		func(in string) int64 {
			re := regexp.MustCompile("^([0-9,]+) - 即時$")
			matches := re.FindStringSubmatch(in)
			// fmt.Println("matches:", matches)
			if len(matches) != 2 {
				return 0
			}

			comma_included_string := matches[1]
			// fmt.Println(comma_included_string)
			replacer := strings.NewReplacer(",", "")
			vali, _ := strconv.ParseInt(replacer.Replace(comma_included_string), 10, 64)
			return vali
		})
	// fmt.Println("stock <- sum:", stock, sum)
	stock <- sum
}

func ScrapeMouser(partno string, stock chan<- int64) {
	// ex: http://jp.mouser.com/Search/Refine.aspx?Keyword=atmega32u4-au
	Url, _ := url.Parse("http://jp.mouser.com/Search/Refine.aspx")
	parameters := url.Values{}
	parameters.Add("Keyword", partno)
	Url.RawQuery = parameters.Encode()
	// fmt.Println("Encoded URL is", Url.String())

	sum := sumNumberInURL(
		Url.String(),
		"tr.SearchResultsRowOdd td:nth-of-type(8) span,tr.SearchResultsRowEven td:nth-of-type(8) span",
		func(in string) int64 {
			re := regexp.MustCompile("^([0-9,]+)在庫$")
			matches := re.FindStringSubmatch(in)
			// fmt.Println("matches:", matches)
			if len(matches) != 2 {
				return 0
			}

			comma_included_string := matches[1]
			// fmt.Println(comma_included_string)
			replacer := strings.NewReplacer(",", "")
			vali, _ := strconv.ParseInt(replacer.Replace(comma_included_string), 10, 64)
			return vali
		})
	// fmt.Println("stock <- sum:", stock, sum)
	stock <- sum
}

func main() {
	if len(os.Args) == 1 {
		fmt.Printf("Usage: stock-scraper partno [partno2 ..]\nOutputs sum of stock in Digikey and Mouser per partno:\nExample output: \"0\t1111\t\"\n")
		os.Exit(1)
	}

	partnos := os.Args[1:]
	scrapers := []scraper{ScrapeDigikey, ScrapeMouser}

	stock := make([]chan int64, len(partnos))
	for i, _ := range partnos {
		stock[i] = make(chan int64, len(scrapers))
	}

	for i, partno := range partnos {
		for _, scraper := range scrapers {
			go scraper(partno, stock[i])
		}
	}

	sum := make([]int64, len(partnos), len(partnos))
	for i, _ := range partnos {
		for _, _ = range scrapers {
			sum[i] += <-stock[i]
		}
		fmt.Printf("%d\t", sum[i])
	}
}