Scraping Implementation Using Channels

scrap.go

// Scraping implementation using channels
// go run scrap.go
package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"sync"
)

type Result struct {
	site string
	page []byte
}

func getPage(link string, c chan Result, wg *sync.WaitGroup) {
	defer wg.Done()

	resp, err := http.Get(link)

	if err != nil {
		fmt.Println("error")
		os.Exit(1)
	}

	defer resp.Body.Close()

	data, _ := ioutil.ReadAll(resp.Body)

	res := new(Result)
	res.site = link
	res.page = data
	c <- *res
}

func main() {
	links := []string{
		"http://www.detik.com",
		"http://www.kompas.com",
		"http://news.metrotvnews.com",
		"http://okezone.com",
		"http://www.liputan6.com",
		"http://www.jawapos.com",
		"http://www.viva.co.id",
		"http://www.republika.co.id/",
		"http://www.tribunnews.com/",
		"http://www.antaranews.com/",
	}

	c := make(chan Result)
	wg := &sync.WaitGroup{}

	for index := range links {
		wg.Add(1)
		go getPage(links[index], c, wg)
	}

	go func() {
		wg.Wait()
		close(c)
	}()

	for res := range c {
		fmt.Println(res.site, ":", string(res.page))
	}

}