go-tour 71 web crawler

71.go

package main

import (
  "errors"
  "fmt"
  "log"
  "math/rand"
  "time"
)

type Fetcher interface {
  // Fetch returns the body of URL and
  // a slice of URLs found on that page.
  Fetch(url string) (body string, urls []string, err error)
}

type fetchConfig struct {
  url     string
  depth   int
  fetcher Fetcher
}

type fetchResult struct {
  config fetchConfig
  body   string
  urls   []string
  err    error
}

func _crawler(configCh chan fetchConfig, resultCh chan fetchResult) {

  config := <-configCh
  body, urls, err := config.fetcher.Fetch(config.url)
  if err != nil {
    fmt.Println(err)
    resultCh <- fetchResult{config, body, urls, err}
  } else {
    fmt.Printf("found: %s %q\n", config.url, body)
    resultCh <- fetchResult{config, body, urls, err}
  }
}

func Crawler(url string, depth int, fetcher Fetcher) {

  configCh := make(chan fetchConfig)
  defer close(configCh)

  resultCh := make(chan fetchResult)
  defer close(resultCh)

  fetched := make(map[string]error)
  fetching := errors.New("fetching")

  go _crawler(configCh, resultCh)
  configCh <- fetchConfig{url, depth, fetcher}
  for result := range resultCh {
    fetched[result.config.url] = result.err
    for _, u := range result.urls {
      if _, ok := fetched[u]; !ok && result.config.depth > 0 {
        fetched[u] = fetching
        go _crawler(configCh, resultCh)
        configCh <- fetchConfig{u, result.config.depth - 1, fetcher}
      }
    }
    fetchCompleted := true
    for _, v := range fetched {
      if fetchCompleted && v == fetching {
        fetchCompleted = false
      }
    }
    if fetchCompleted {
      return
    }
  }
}

func main() {
  Crawler("http://golang.org/", 4, fetcher)
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
  body string
  urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
  if res, ok := f[url]; ok {
    return res.body, res.urls, nil
  }
  return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
  "http://golang.org/": &fakeResult{
    "The Go Programming Language",
    []string{
      "http://golang.org/pkg/",
      "http://golang.org/cmd/",
    },
  },
  "http://golang.org/pkg/": &fakeResult{
    "Packages",
    []string{
      "http://golang.org/",
      "http://golang.org/cmd/",
      "http://golang.org/pkg/fmt/",
      "http://golang.org/pkg/os/",
    },
  },
  "http://golang.org/pkg/fmt/": &fakeResult{
    "Package fmt",
    []string{
      "http://golang.org/",
      "http://golang.org/pkg/",
    },
  },
  "http://golang.org/pkg/os/": &fakeResult{
    "Package os",
    []string{
      "http://golang.org/",
      "http://golang.org/pkg/",
    },
  },
}