Skip to content

Instantly share code, notes, and snippets.

@hjhee
Created July 27, 2017 07:16
Show Gist options
  • Save hjhee/9cfc3bbd74c3f7bdcdcf5783d8a9c3e4 to your computer and use it in GitHub Desktop.
Save hjhee/9cfc3bbd74c3f7bdcdcf5783d8a9c3e4 to your computer and use it in GitHub Desktop.
solution with package "sync"
// https://tour.go-zh.org/concurrency/10
func Crawl(url string, depth int, fetcher Fetcher) {
var wg sync.WaitGroup
var mux sync.Mutex
fetched := make(map[string]bool)
var crawl func(url string, depth int, fetcher Fetcher)
crawl = func (url string, depth int, fetcher Fetcher) {
defer wg.Done() // ref - 1
if depth <= 0 {
return
}
mux.Lock() // mutex
if !fetched[url] {
fetched[url] = true // set flag
mux.Unlock()
} else { // fetch already started or done
mux.Unlock()
return
}
body, urls, err := fetcher.Fetch(url) // start fetch
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("found: %s %q\n", url, body)
for _, u := range urls {
wg.Add(1) // ref + 1
url := u // necessary?
go crawl(url, depth-1, fetcher)
}
}
wg.Add(1)
crawl(url, depth, fetcher)
wg.Wait()
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment