A Tour of Go #70 Exercise: Web Crawler

crawler.go

package main

import (
    "fmt"
)

type Fetcher interface {
    Fetch(url string) (body string, urls []string, err error)
}

///////////////////////////////////////////////////////////////////

type Result struct{
    Url string
    Body string
    Error error
}
var fetched = make(map[string]bool)
var num_results = 0
var ch = make(chan Result)

func Crawl(url string, depth int, fetcher Fetcher) {
    if depth <= 0 {
        return
    }
    body, urls, err := fetcher.Fetch(url)
    fetched[url] = true
    ch <- Result{url, body, err}
    for _, u := range urls {
        if !fetched[u] {
            num_results++
            go Crawl(u, depth-1, fetcher)
        }
    }
    return
}

func main() {
    num_results++
    go Crawl("http://golang.org/", 4, fetcher)
    for num_results > 0 {
        t := <-ch
        num_results--
        if t.Error!=nil {
            fmt.Println(t.Error)
        } else {
            fmt.Printf("found: %s %q\n", t.Url, t.Body)
        }
    }
}

///////////////////////////////////////////////////////////////////

type fakeFetcher map[string]*fakeResult

type fakeResult struct {
    body string
    urls []string
}

func (f *fakeFetcher) Fetch(url string) (string, []string, error) {
    if res, ok := (*f)[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

var fetcher = &fakeFetcher{
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}