gonx vs glow

glow_bench.go

package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"strings"

	"github.com/chrislusf/glow/flow"
)

func main() {
	flag.Parse()

	flow.New().TextFile(
		"top-1m.csv", 3,
	).Map(func(line string, ch chan string) {
		parts := strings.Split(line, ",")
		data := []byte(parts[1])
		ch <- fmt.Sprintf("%x", md5.Sum(data))
	}).Map(func(s string) {
		fmt.Println(s)
	}).Run()
}

gonx_bench.go

package main

import (
	"crypto/md5"
	"fmt"
	"os"
	"strings"

	"github.com/satyrius/gonx"
)

type SplitParser struct{}

func (p *SplitParser) ParseString(line string) (*gonx.Entry, error) {
	parts := strings.Split(line, ",")
	entry := gonx.NewEmptyEntry()
	entry.SetField("domain", parts[1])
	return entry, nil
}

type Hash struct {
	Fields []string
}

func (r *Hash) Reduce(input chan *gonx.Entry, output chan *gonx.Entry) {
	for entry := range input {
		for _, name := range r.Fields {
			val, err := entry.Field(name)
			if err == nil {
				data := []byte(val)
				s := fmt.Sprintf("%x", md5.Sum(data))
				entry.SetField(name, s)
			}
		}
		output <- entry
	}
	close(output)
}

func main() {
	reader, _ := os.Open("top-1m.csv")
	parser := new(SplitParser)
	reducer := &Hash{Fields: []string{"domain"}}
	output := gonx.MapReduce(reader, parser, reducer)
	for res := range output {
		val, err := res.Field("domain")
		if err == nil {
			fmt.Println(val)
		}
	}
}