Go semaphore worker pattern

main.go

// this is a simple example of a semaphore in go.
//
// it replaces the typical waitgroup pattern. The benefit of a semaphore is that
// no workers will be left abandoned and no unneeded workers are started in
// contrast to a typical waitgroup based worker pool pattern. We also can only
// start a limited amount of workers at once. In a waitgroup we coul start more
// workers than we have tasks, which is not always optimal.

package main

import (
	"log"
	"time"
)

// token that is acquired and returned on our semaphore
type token struct{}

// Task is the data for the job that has to run
type Task struct {
	input int
}

// how many goroutines can run at once
var limit int = 10

func main() {
	// our tasks comes in on this channel
	tasks := make(chan Task)

	// create a data generator in a separate goroutine, this is not part of the
	// pattern but only generates usable data input
	go func() {
		for i := 0; i < 10; i++ {
			tasks <- Task{input: i}
		}

		// close the channel as we are done with sending data
		close(tasks)
	}()

	// this is our semaphore
	sem := make(chan token, limit)

	for task := range tasks {
		// lent out a token, which means we add a token to the semaphore. There
		// is a max of `limit` tokens in the semaphore. If all tokens are lent,
		// we cannot start any more workers. When the worker finishes, it
		// returns the token space to the semaphore.
		sem <- token{}
		log.Println("token lent")
		go func(t Task) {
			// do the work
			doWork(t)
			// return a token as we are done with the work
			<-sem
			log.Println("token freed")
		}(task)
	}

	// wait for our workers to finish by filling up the semaphore. This will
	// block until all workers have returned their token.
	//
	// In detail: we are filling up the semaphore with tokens, but we have to
	// wait for free spaces. A space becomes free once a worker has finished and
	// returns their token space to the semaphore.
	for n := limit; n > 0; n-- {
		sem <- token{}
		log.Println("worker ended")
	}

	log.Println("all workers done")
}

// doWork could do anything, it is just simulating our work
func doWork(t Task) {
	log.Printf("working with data: %d\n", t.input)
	time.Sleep(250 * time.Millisecond)
}