crawler_cgo_resolver_mem_leak.go

package main

//#include <unistd.h>
import "C"

import (
	"context"
	"log"
	"math/rand"
	"runtime"
	"sync"
	"time"
)
///////////////////////////////////
// Application
///////////////////////////////////
func main() {
	var wg sync.WaitGroup
	workersNum:=1000 // 10000
	for i := 0; i <= workersNum; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			worker()
		}()
	}
	
	for {
		PrintMemUsage()
		time.Sleep(30*time.Second)
	}
	
	wg.Wait()
}

func worker() {
	for i := 0; i < 100000; i++ {
		// http.Get with Second timeout
		ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second)
		doSmth(ctx)
		ctxCancel()
	}
}

///////////////////////////////////
// CGO resolver emulation
///////////////////////////////////

// see net.go var threadLimit chan struct{}
var threadsLimiter chan int
func init() {
	threadsLimiter = make(chan int, 500 /*OS threads limit*/)
}

// see go/src/net/cgo_unix.go
// func cgoIPLookup & cgoLookupIPCNAME
func cgoCall(res chan struct{}) {
	threadsLimiter <- 1
	defer func() {
		<-threadsLimiter
	}()
	t := uint(rand.Int() % 100)
	C.sleep(C.uint(t)) // ~ C.getaddrinfo
	close(res)
}

// see cgo_unix.go: func cgoLookupIP
func doSmth(ctx context.Context) bool {
	res := make(chan struct{})

	go cgoCall(res) // <---- goroutine leak (actually, there is no leak, we just can't control goroutine lifetime)

	select {
	case <-res:
		return true
	case <-ctx.Done():
		return false
	}
}

/////////////////////////
// Resources usage utils
/////////////////////////
func bToMb(b uint64) uint64 {
	return b / 1024 / 1024
}

func PrintMemUsage() {
	var m runtime.MemStats
	runtime.ReadMemStats(&m)
	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
	log.Printf("Alloc = %v MiB", bToMb(m.Alloc))
	log.Printf("TotalAlloc = %v MiB", bToMb(m.TotalAlloc))
	log.Printf("Sys = %v MiB", bToMb(m.Sys))
	log.Printf("NumGoroutine = %v", runtime.NumGoroutine())
	log.Print("")
}