Generates "random" documents, each with a batch ID, in a parallel way using a fan-out pattern.

generate_mongo_data.go

package main

import (
  "runtime"
  "time"
  "math/rand"
  "fmt"
  "labix.org/v2/mgo"
)

const NUM_ELEMENTS      = 10000
const NUM_TIMES         = 1

func randomData() (map[string] string) {
  m := make(map[string] string)
  n := rand.Intn(12)

  for i := 0; i < n; i++ {
    m[fmt.Sprintf("key%d", i)] = fmt.Sprintf("value%d", i)
  }

  return m
}

func insertToMongo(batchName string, ch chan int) {
  session, err := mgo.Dial("localhost:27017")

  if err != nil {
    panic(err)
  }

  // We're good to go. Let's generate a shit load of data.
  defer session.Close()

  session.SetMode(mgo.Monotonic, true)

  coll := session.DB("generated_data").C("go_test1")

  for i := 0; i < NUM_ELEMENTS; i++ {
    data := randomData()
    data["batch_id"] = batchName

    coll.Insert(data)
  }

  ch <- NUM_ELEMENTS
}

func main() {
  ch := make(chan int)
  sum, max_routines := 0, runtime.NumCPU()

  start_at := time.Now()

  for i := 0; i < NUM_TIMES; i++ {
    for j := 0; j < max_routines; j++ {
      go insertToMongo(fmt.Sprintf("batch%d", (j * max_routines) + i), ch)
    }

    fmt.Printf("Inserting %d documents.\n", NUM_ELEMENTS * max_routines)

    count := 0
    for count < max_routines {
      sum += <-ch
      fmt.Printf("* Inserted %d documents.\n", sum)
      count++
    }
  }

  end_at := time.Now()
  duration := end_at.Sub(start_at)
  fmt.Printf("Took %0.03fms\n", duration.Seconds() * 1000.0)
}