test-pipe.go

package main

// produces 10KB json (newline delimited) to stdout
// ref: https://github.com/mozilla-services/Dockerflow/issues/31

import (
	"fmt"
	"runtime"
	"strings"
	"sync"
	"time"
)

var empty = struct{}{}

func main() {

	var wg sync.WaitGroup

	done := make(chan struct{})

	// run a bunch of parallel writers
	for i := 0; i < runtime.NumCPU()*2; i++ {
		go func(id int) {
			defer wg.Done()
			// 10KB string blob
			dataBlob := strings.Repeat("a", 10*1024)

			// keep outputting 10KB json blobs every 100ms
			// until time is up.
			for {
				select {
				case <-time.After(100 * time.Millisecond):
					out := fmt.Sprintf(`{"id":%d,"data":"%s"}`, id, dataBlob)
					fmt.Println(out)
				case <-done:
					return
				}
			}
		}(i)

		wg.Add(1)
	}

	// times up in 3 seconds
	go func() {
		time.Sleep(3 * time.Second)
		close(done)
	}()

	wg.Wait()
}

Usage

> go run test-pipe.go  | jq -rc '"id:\(.id) -> \(.data | length) bytes"'
id:4 -> 10240 bytes
id:5 -> 10240 bytes
id:6 -> 10240 bytes
id:7 -> 10240 bytes
... (3 seconds of output)
id:5 -> 10240 bytes