"A Tour of Go"

WebCrawler.go

/*
Exercise: Web Crawler

In this exercise you'll use Go's concurrency features to parallelize a web crawler.

Modify the Crawl function to fetch URLs in parallel without fetching the same URL twice.
*/

package main

import (
    "fmt"
    "sync"
    "errors"
)

type Fetcher interface {
    // Fetch returns the body of URL and
    // a slice of URLs found on that page.
    Fetch(url string) (body string, urls []string, err error)
}

type Fetched struct {
    record map[string]error
    sync.Mutex
}
var fetched = &Fetched{record:make(map[string]error)}

var loading = errors.New("url load in process")
                         
// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {
    // TODO: Fetch URLs in parallel.
    // TODO: Don't fetch the same URL twice.
    // This implementation doesn't do either:
    if depth <= 0 {
        return
    }
    fetched.Lock()
    if _,ok := fetched.record[url]; ok {
        fetched.Unlock()
        fmt.Printf("<- Done with %v, already fetched.\n",url)
        return
    }
    fetched.record[url] = loading
    fetched.Unlock()
    
    body, urls, err := fetcher.Fetch(url)
    fetched.Lock()
    fetched.record[url] = err
    fetched.Unlock()
    
    if err != nil {
        fmt.Printf("<- Error on [%v]:%v\n",url,err)
        return
    }
    fmt.Printf("found: %s %q\n", url, body)
    done := make(chan bool)
    for _, u := range urls {
        go func(url string) {
            Crawl(url, depth-1, fetcher)
            done <- true
        }(u)
    }
    for i,u:= range urls {
        fmt.Printf("<- [%v] %v/%v Waiting for child %v.\n", url, i, len(urls),u)
        <-done
    } 
    fmt.Printf("<- Done with %v\n", url)
    return
}

func main() {

    Crawl("http://golang.org/", 4, fetcher)
    
    fmt.Println("Fetching stats\n--------------")
    for url, err := range fetched.record {
        if err != nil {
            fmt.Printf("%v failed: %v\n", url, err)
        } else {
            fmt.Printf("%v was fetched\n", url)
        }
    }
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
    body string
    urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
    if res, ok := f[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}