Created
October 2, 2013 12:41
-
-
Save firegoby/6793063 to your computer and use it in GitHub Desktop.
Solution to A Tour of Go final exercise - modify a web crawler to fetch URLs in parallel without fetching the same URL twice. Use Go's concurrency features. I used sync.WaitGroup to control the goroutines and a simple string->bool map to store already fetched URLs. Inline comments have further info
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"fmt" | |
"sync" | |
) | |
type Fetcher interface { | |
// Fetch returns the body of URL and | |
// a slice of URLs found on that page. | |
Fetch(url string) (body string, urls []string, err error) | |
} | |
var fetched = make(map[string]bool) // keep track of fetched URLs | |
var wg sync.WaitGroup // global WaitGroup | |
// Crawl uses fetcher to recursively crawl | |
// pages starting with url, to a maximum of depth. | |
func Crawl(url string, depth int, fetcher Fetcher) { | |
defer wg.Done() // notify WaitGroup when we're finished | |
if depth <= 0 { | |
return | |
} | |
if fetched[url] == true { // only need to fetch each URL once | |
return | |
} | |
body, urls, err := fetcher.Fetch(url) | |
fetched[url] = true // mark it fetched | |
// NOTE: If above is placed after the err check then not-found URLs will be | |
// tried to be fetched again the next time they're encountered, having it | |
// before means that once they've 404'd once, they're considered done with | |
if err != nil { | |
fmt.Println(err) | |
return | |
} | |
fmt.Printf("found: %s %q\n", url, body) | |
for _, u := range urls { | |
wg.Add(1) // notify WaitGroup of a new goroutine... | |
go Crawl(u, depth-1, fetcher) // ... and dispatch it | |
} | |
return | |
} | |
func main() { | |
wg.Add(1) // add initial goroutine to WaitGroup | |
go Crawl("http://golang.org/", 4, fetcher) | |
wg.Wait() // await completion of all goroutines | |
} | |
// fakeFetcher is Fetcher that returns canned results. | |
type fakeFetcher map[string]*fakeResult | |
type fakeResult struct { | |
body string | |
urls []string | |
} | |
func (f fakeFetcher) Fetch(url string) (string, []string, error) { | |
if res, ok := f[url]; ok { | |
return res.body, res.urls, nil | |
} | |
return "", nil, fmt.Errorf("not found: %s", url) | |
} | |
// fetcher is a populated fakeFetcher. | |
var fetcher = fakeFetcher{ | |
"http://golang.org/": &fakeResult{ | |
"The Go Programming Language", | |
[]string{ | |
"http://golang.org/pkg/", | |
"http://golang.org/cmd/", | |
}, | |
}, | |
"http://golang.org/pkg/": &fakeResult{ | |
"Packages", | |
[]string{ | |
"http://golang.org/", | |
"http://golang.org/cmd/", | |
"http://golang.org/pkg/fmt/", | |
"http://golang.org/pkg/os/", | |
}, | |
}, | |
"http://golang.org/pkg/fmt/": &fakeResult{ | |
"Package fmt", | |
[]string{ | |
"http://golang.org/", | |
"http://golang.org/pkg/", | |
}, | |
}, | |
"http://golang.org/pkg/os/": &fakeResult{ | |
"Package os", | |
[]string{ | |
"http://golang.org/", | |
"http://golang.org/pkg/", | |
}, | |
}, | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment