client.go

package client

import (
  "net/http"
  "io"
)

import _ "code.google.com/p/go-charset/data"

type ClientCallback func(*Client, Response)

type Response struct {
  *http.Response
  Request *Request
  Error error
}

type Request struct {
  *http.Request
  Callback ClientCallback
}

type Client struct {
  Requests []chan *Request
  Responses chan *Response
  n int
  pending int
  closed bool
}

func downloader(requests chan *Request, responses chan *Response){
  client := http.Client{}

  for req := range(requests) {
    resp := &Response{Request: req}

    if reply, err := client.Do(req.Request); err != nil {
      resp.Error = err
    }else {
      resp.Response = reply
    }

    responses <- resp
  }
}

func NewClient(downloaders int) *Client {
  ret := &Client{
    Requests: make([]chan *Request, downloaders),
    Responses: make(chan *Response),
    n: 0,
    pending: 0,
  }

  for i := 0; i < downloaders; i++ {
    ret.Requests[i] = make(chan *Request)
    go downloader(ret.Requests[i], ret.Responses)
  }

  return ret
}

func (v *Client) NewRequest(method string, url string, body io.Reader, cb ClientCallback) (*Request, error) {
  if r, e := http.NewRequest(method, url, body); e != nil {
    return nil, e
  } else {
    return &Request{Request: r, Callback: cb}, nil
  }
}

func (v *Client) Do(req *Request){
  v.Requests[v.n] <- req
  v.pending += 1

  v.n += 1
  if v.n >= len(v.Requests) {
    v.n = 0
  }
}

func (v *Client) Get(url string, cb ClientCallback) error {
  if r, e := v.NewRequest("GET", url, nil, cb); e != nil {
    return e
  } else {
    v.Do(r)
    return nil
  }
}

func (v *Client) GetReferer(url string, referer string, cb ClientCallback) error {
  req, e := v.NewRequest("GET", url, nil, cb)

  if e != nil {
    return e
  }

  req.Header.Add("Referer", referer)

  v.Do(req)
  return nil
}

func (v *Client) processResponse(res *Response){
  res.Request.Callback(v, *res)
  v.pending -= 1
}

func (v *Client) ProcessResponse(block bool){
  if block {
    v.processResponse(<- v.Responses)
  } else {
    select {
      case res := <- v.Responses:
        v.processResponse(res)
      default:
    }
  }
}

func (v *Client) Run(){
  for v.pending > 0 && !v.closed {
    v.ProcessResponse(true)
  }

  close(v.Responses)

  if !v.closed {
    v.Close()
  }
}

func (v *Client) Close(){
  for i := 0; i < len(v.Requests); i += 1 {
    close(v.Requests[i])
  }

  v.closed = true
}

main.go

package main

import (
  "fmt"
  "naquad.me/client"
  "github.com/PuerkitoBio/goquery"
)

func parseFront(client *client.Client, res client.Response){
  doc, e := goquery.NewDocumentFromResponse(res.Response)

  fmt.Println(res.Request.URL)

  if e != nil {
    fmt.Println(e)
    return
  }

  for _, e := range(doc.Find(".main-nav li a[href]").Nodes) {
    fmt.Println(e)
  }
}

func parseCategory(client *client.Client, res client.Response){
  doc, e := goquery.NewDocumentFromResponse(res.Response)

  if e != nil {
    fmt.Println(e)
    return
  }

  for _, e := range(doc.Find("#category-map a").Nodes) {
    fmt.Println(e)
  }
}

func parseList(client *client.Client, res client.Response){
  doc, e := goquery.NewDocumentFromResponse(res.Response)

  if e != nil {
    fmt.Println(e)
    return
  }

  for _, e := range(doc.Find("#listing .pager-bottom .next a[href]").Nodes) {
    fmt.Println(e)
  }

  for _, e := range(doc.Find("#listing .offers article[id^='item-'] h2 a").Nodes) {
    fmt.Println(e)
  }
}

func parseProduct(client *client.Client, res client.Response){
  
}

func main(){
  client := client.NewClient(30)
  client.Get("http://www.aukro.ua", parseFront)
  client.Run()
}