main.go

package main

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"sync"

	"crawler-sample/title"
)

type WebPage struct {
	URL   string
	Body  io.Reader
	Title string
}

func download(webPage *WebPage) error {
	resp, err := http.Get(webPage.URL)
	if err == nil {
		webPage.Body = resp.Body
	}
	return err
}

func parse(webPage *WebPage) error {
	htmlTitle, ok := title.GetHtmlTitle(webPage.Body)
	if ok {
		webPage.Title = htmlTitle
		return nil
	} else {
		return errors.New("fail on parse")
	}
}

func present(webPage *WebPage) {
	fmt.Println(webPage.Title)
}

func main() {
	urls := []string{
		"https://www.google.co.jp/",
		"https://www.google.co.jp/intl/ja/policies/privacy/",
		"http://ssig33.com/",
		"http://ssig33.com/text",
		"http://ssig33.com/text/%E6%97%A5%E8%A8%98%E3%81%A7%E3%81%99",
		"https://qiita.com/",
		"https://qiita.com/ss_watanabe/items/dd1bdd9b01ee9a8c9eaa",
		"https://www.2nn.jp/",
		"https://golang.org/pkg/errors/",
		"https://qiita.com/jpshadowapps/items/463b2623209479adcd88",
		"http://diary.app.ssig33.com/",
	}

	c := make(chan bool, 5)
	var wg sync.WaitGroup

	for _, url := range urls {
		c <- true
		wg.Add(1)

		go func(url string) {
			defer func() { <-c }()
			defer wg.Done()

			webPage := WebPage{URL: url}

			err := download(&webPage)

			if err == nil {
				err := parse(&webPage)
				if err == nil {
					present(&webPage)
				}
			}
		}(url)
	}

	wg.Wait()
}