Using Go for embarrassingly parallel scripts

domain_lookup_parallel.go

package main

import (
  "fmt"
  "net"
  "io/ioutil"
  "strings"
)

type DomainMap struct {
  Domain string
  IpMapping string
}

func retrieveDomains() []string {
  file_in, _ := ioutil.ReadFile("domains.txt")
  domain_list := string(file_in)
  return strings.Split(strings.TrimSpace(domain_list), "\n")
}

func domainLookup(returnChannel chan DomainMap, domain string) {
  rawIpAddresses, _ := net.LookupIP(domain)

  // If any found, grab only the first address for simplicity as the ip lookup can return an array
  ipAddress := ""
  if len(rawIpAddresses) > 0 {
    ipAddress = rawIpAddresses[0].String()
  }

  fmt.Println("Mapping: ", domain, "->", ipAddress)

  // Send our results back to the main processes via our return channel
  returnChannel <- DomainMap{domain, ipAddress}
}

// The extra set of parentheses here are the return type. You can give the return value a name,
// in this case +domainMapping+ and use that name in the function body. Then you don't need to specify
// what actually gets returned, you've already defined it here.
func waitForDomains(responseChannel chan DomainMap, numberOfDomains int) (domainMapping []DomainMap) {
  returnedCount := 0
  for {
    domainMapping = append(domainMapping, <- responseChannel)
    returnedCount++

    if returnedCount >= numberOfDomains {
      break
    }
  }

  return
}

func main() {
  domains := retrieveDomains()

  // This is the channel the responses will come back on
  responseChannel := make(chan DomainMap)

  // Send our requests, one for each domain we get in their own goroutine
  for _, domain := range domains {
    go domainLookup(responseChannel, domain)
  }

  // Wait for all the goroutines to finish, collecting the responses
  domainMapping := waitForDomains(responseChannel, len(domains))

  fmt.Println(domainMapping)
}

domain_lookup_sequential.go

package main

import (
  "fmt"
  "net"
  "io/ioutil"
  "strings"
)

type DomainMap struct {
  Domain string
  IpMapping string
}

func retrieveDomains() []string {
  file_in, _ := ioutil.ReadFile("domains.txt")
  domain_list := string(file_in)
  return strings.Split(strings.TrimSpace(domain_list), "\n")
}

func domainLookup(domain string) DomainMap {
  rawIpAddresses, _ := net.LookupIP(domain)

  // If any found, grab only the first address for simplicity as the ip lookup can return an array
  ipAddress := ""
  if len(rawIpAddresses) > 0 {
    ipAddress = rawIpAddresses[0].String()
  }

  fmt.Println("Mapping: ", domain, "->", ipAddress)

  return DomainMap{domain, ipAddress}
}

func main() {
  domains := retrieveDomains()
  var domainMapping []DomainMap

  // Send our requests, one for each domain we get in their
  // own goroutine
  for _, domain := range domains {
    domainMapping = append(domainMapping, domainLookup(domain))
  }

  fmt.Println(domainMapping)
}

Timings.txt

] wc -l domains.txt 
     783 domains.txt

] time go run domain_lookup_parallel.go

real	0m5.743s
user	0m0.359s
sys	0m0.355s

] time go run domain_lookup_sequential.go

real	0m43.794s
user	0m0.320s
sys	0m0.200s