My somewhat convoluted approach to the web crawler exercise in the Tour of Go http://tour.golang.org/#70. Uses ideas from http://soniacodes.wordpress.com/2011/10/09/a-tour-of-go-69-exercise-web-crawler/ with regards to how to serialize access to the map, and Rob Pike's Go Currurency Pattern's talk http://www.youtube.com/watch?v=f6kdp27TYZs with …

crawl.go

package main

import (
	"fmt"
)

type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page.
	Fetch(url string) (body string, urls []string, err error)
}

type Result struct {
	url   string
	body  string
	links []string
	err   error
}

func fetch(url string, fetcher Fetcher) Result {
	body, links, err := fetcher.Fetch(url)
	if err != nil {
		return Result{url, "", nil, err}
	} else {
		return Result{url, body, links, nil}
	}
}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) <-chan Result {
	mapAccess := make(chan map[string]bool, 1)
	mapAccess <- make(map[string]bool)

	var loop func(string, int) <-chan Result
	loop = func(url string, depth int) <-chan Result {
		c := make(chan Result)
		go func() {
			if depth <= 0 {
				close(c)
				return
			}

			// Check whether we've already visited this.
			m := <-mapAccess
			if _, in := m[url]; in {
				close(c)
				mapAccess <- m
				return
			}
			m[url] = true
			mapAccess <- m

			r := fetch(url, fetcher)
			if r.err != nil {
				c <- r
				close(c)
				return
			}

			c <- r

			child_chs := make([]<-chan Result, 0)
			for _, u := range r.links {
				child_chs = append(child_chs, loop(u, depth-1))
			}
			for r := range multiplex(child_chs) {
				c <- r
			}
			close(c)
		}()
		return c
	}
	return loop(url, depth)
}

func multiplex(chs []<-chan Result) <-chan Result {
	c := make(chan Result)
	d := make(chan bool)
	for _, ch := range chs {
		go func(ch <-chan Result) {
			for r := range ch {
				c <- r
			}
			d <- true
		}(ch)
	}
	go func() {
		for i := 0; i < len(chs); i++ {
			<-d
		}
		close(c)
	}()
	return c
}

func main() {
	c := Crawl("http://golang.org/", 4, fetcher)
	for r := range c {
		if r.err != nil {
			fmt.Println(r.err)
		} else {
			fmt.Printf("found: %s %q\n", r.url, r.body)
		}
	}
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
	body string
	urls []string
}

func (f *fakeFetcher) Fetch(url string) (string, []string, error) {
	if res, ok := (*f)[url]; ok {
		return res.body, res.urls, nil
	}
	return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = &fakeFetcher{
	"http://golang.org/": &fakeResult{
		"The Go Programming Language",
		[]string{
			"http://golang.org/pkg/",
			"http://golang.org/cmd/",
		},
	},
	"http://golang.org/pkg/": &fakeResult{
		"Packages",
		[]string{
			"http://golang.org/",
			"http://golang.org/cmd/",
			"http://golang.org/pkg/fmt/",
			"http://golang.org/pkg/os/",
		},
	},
	"http://golang.org/pkg/fmt/": &fakeResult{
		"Package fmt",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
	"http://golang.org/pkg/os/": &fakeResult{
		"Package os",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
}