Go implementations for http://ileriseviye.wordpress.com/2014/10/31/how-to-get-better-performance-from-scala-by-using-parallel-collections/

dl_1.go

package main

import (
	"io/ioutil"
	"net/http"
	"os"
	fp "path/filepath"
	re "regexp"
	"sync"
)

func main() {
	p, e := re.Compile("(http://www.standaard.be/cnt/dmf.*?)\"")
	check(e)

	r, e := http.Get("http://www.standaard.be/nieuws/chronologisch")
	check(e)
	b, e := ioutil.ReadAll(r.Body)
	check(e)

	ls := p.FindAll(b, 10000)

	wg := new(sync.WaitGroup)
	wg.Add(len(ls))
	for _, l := range ls {
		s := string(l)
		go func() {
			c, _ := http.Get(s)
			b, e = ioutil.ReadAll(c.Body)
			check(e)
			ioutil.WriteFile(fp.Join("/tmp/a/", fp.Base(s)), b, os.ModePerm)
			wg.Done()
		}()
	}
	wg.Wait()
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

dl_2.go

package main

import (
	"io/ioutil"
	"net/http"
	"os"
	fp "path/filepath"
	re "regexp"
	"sync"
)

func main() {
	p, e := re.Compile("(http://www.standaard.be/cnt/dmf.*?)\"")
	check(e)

	r, e := http.Get("http://www.standaard.be/nieuws/chronologisch")
	check(e)
	b, e := ioutil.ReadAll(r.Body)
	check(e)

	ls := p.FindAll(b, 10000)

	wg := new(sync.WaitGroup)
	wg.Add(8)

	ps := len(ls) / 8

	for i := 0; i < 7; i++ {
		go dl(ls[:ps], wg)
		ls = ls[ps:]
	}
	dl(ls, wg)
	wg.Wait()
}

func dl(ls [][]byte, wg *sync.WaitGroup) {
	for _, l := range ls {
		s := string(l)
		c, _ := http.Get(s)
		b, e := ioutil.ReadAll(c.Body)
		check(e)
		ioutil.WriteFile(fp.Join("/tmp/a/", fp.Base(s)), b, os.ModePerm)
	}
	wg.Done()
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

dl_3.go

package main

import (
	"io/ioutil"
	"net/http"
	"os"
	fp "path/filepath"
	re "regexp"
)

func main() {
	p, e := re.Compile("(http://www.standaard.be/cnt/dmf.*?)\"")
	check(e)

	r, e := http.Get("http://www.standaard.be/nieuws/chronologisch")
	check(e)
	b, e := ioutil.ReadAll(r.Body)
	check(e)

	ls := p.FindAll(b, 10000)

	for _, l := range ls {
		s := string(l)
		c, _ := http.Get(s)
		b, e = ioutil.ReadAll(c.Body)
		check(e)
		ioutil.WriteFile(fp.Join("/tmp/a/", fp.Base(s)), b, os.ModePerm)
	}
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

Results.txt

For 1 Processor:


dl_1:
real	0m1.081s
user	0m0.112s
sys	0m0.084s

dl_2:
real	0m1.016s
user	0m0.092s
sys	0m0.056s

dl_3:
real	0m4.330s
user	0m0.104s
sys	0m0.088s



For 8 processors:

dl_1:
real	0m1.263s
user	0m0.092s
sys	0m0.164s


dl_2:
real	0m1.176s
user	0m0.124s
sys	0m0.108s

dl_3:
real	0m4.323s
user	0m0.100s
sys	0m0.144s