Exercise: Web Crawler (https://tour.golang.org/concurrency/9)

exercise-web-crawler.go

package main

import (
  "fmt"
)

type Fetcher interface {
  // Fetch returns the body of URL and
  // a slice of URLs found on that page.
  Fetch(url string) (body string, urls []string, err error)
}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {

  var crawl func(url string, depth int, v chan map[string]bool, done chan bool)
  crawl = func(url string, depth int, v chan map[string]bool, done chan bool) {
    if depth <= 0 {
      done <- true
      return
    }

    m := <- v
    if _, ok := m[url]; ok {
      v <- m
      done <- true
      return
    }
    m[url] = true
    v <- m
	
    body, urls, err := fetcher.Fetch(url)
    if err != nil {
      fmt.Println(err)
      done <- true
      return
    }
    fmt.Printf("found: %s %q\n", url, body)
    cnt := len(urls)
    ch := make(chan bool, cnt)
    for _, u := range urls {
      go crawl(u, depth-1, v, ch)
    }
    for cnt > 0 {
      select {
      case <- ch:
        cnt--
      }
    }
    done <- true
    return
  }
	
  visited := make(chan map[string]bool, 1)
  visited <- map[string]bool{}
  done := make(chan bool, 1)
  crawl(url, depth, visited, done)
  for {
    select {
    case <- done:
      return
    }
  }
}

func main() {
  Crawl("http://golang.org/", 4, fetcher)
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
  body string
  urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
  if res, ok := f[url]; ok {
    return res.body, res.urls, nil
  }
  return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
  "http://golang.org/": &fakeResult{
    "The Go Programming Language",
    []string{
      "http://golang.org/pkg/",
      "http://golang.org/cmd/",
    },
  },
  "http://golang.org/pkg/": &fakeResult{
    "Packages",
    []string{
      "http://golang.org/",
      "http://golang.org/cmd/",
      "http://golang.org/pkg/fmt/",
      "http://golang.org/pkg/os/",
    },
  },
  "http://golang.org/pkg/fmt/": &fakeResult{
    "Package fmt",
    []string{
      "http://golang.org/",
      "http://golang.org/pkg/",
    },
  },
  "http://golang.org/pkg/os/": &fakeResult{
    "Package os",
    []string{
      "http://golang.org/",
      "http://golang.org/pkg/",
    },
  },
}