main.go

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/adtalos/gocandy/must"
	"github.com/gammazero/workerpool"
)

type batchItem struct {
	request request
	gdtids  []int64
}

var client = &http.Client{
	Transport: &http.Transport{
		MaxIdleConnsPerHost:   50,
		IdleConnTimeout:       time.Minute,
		TLSHandshakeTimeout:   time.Second * 10,
		ExpectContinueTimeout: time.Second,
	},
}

func main() {
	for nc, nongdtid := range nongdtids {
		nongdtid, nongdtname := nongdtid, namesByID[nongdtid]

		step := 64
		count := len(gdtids) / step
		if len(gdtids)%step > 0 {
			count++
		}
		batch := make([]batchItem, count)

		for i := range batch {
			batch[i].request = request{
				DeviceidTypes: []string{"imei", "androidid", "oaid", "idfa"},
				Dates:         []string{"2020-05-26"},
			}
		}

		for i, gdtid := range gdtids {
			index := i / step
			batch[index].gdtids = append(batch[index].gdtids, gdtid)
			batch[index].request.Groups = append(batch[index].request.Groups, group{
				A: []int64{nongdtid},
				B: []int64{gdtid},
			})
		}

		buf := bytes.NewBuffer(nil)
		var mu sync.Mutex
		wp := workerpool.New(8)
		for _, batchItem := range batch {
			batchItem := batchItem

			wp.Submit(func() {
				data := bytes.NewBuffer(must.GetBytes(json.Marshal(batchItem.request)))
				resp, err := client.Post("http://whatever-server-go.mobrtb.com/deviceid-bitmaps", "application/json", data)
				if err != nil {
					log.Println(err)
					return
				}
				defer resp.Body.Close()
				if resp.StatusCode != http.StatusOK {
					log.Println("status code is", resp.StatusCode)
					return
				}
				var result response
				err = json.NewDecoder(resp.Body).Decode(&result)
				if err != nil {
					log.Println("failed to decode json", err)
					return
				}
				for i, v := range result.Resultsets {
					gdtid := batchItem.gdtids[i]
					gdtname := namesByID[gdtid]

					if ratio := float64(v.CardinalityIntersection) / float64(v.CardinalityA) * 100; ratio > 10 {
						mu.Lock()
						buf.WriteString(fmt.Sprintf("%d %d-%s %d %d %f\n", v.CardinalityA, gdtid, gdtname, v.CardinalityB, v.CardinalityIntersection, ratio))
						mu.Unlock()
					}
				}
			})
		}
		wp.StopWait()
		if buf.Len() > 0 {
			filename := fmt.Sprintf("/root/analyze/%d-%s", nongdtid, nongdtname)
			f, err := os.Create(filename)
			must.BeNilError(err)
			buf.WriteTo(f)
			must.BeNilError(f.Close())
			log.Println("write to ", filename)
		}

		log.Println("process", nc)
	}
}

type group struct {
	A []int64 `json:"a"`
	B []int64 `json:"b"`
}

type request struct {
	DeviceidTypes []string `json:"deviceid_types"`
	Dates         []string `json:"dates"`
	Groups        []group  `json:"groups"`
}

type resultset struct {
	CardinalityA            uint64 `json:"cardinality_a"`
	CardinalityB            uint64 `json:"cardinality_b"`
	CardinalityIntersection uint64 `json:"cardinality_intersection"`
}

type response struct {
	Resultsets []resultset `json:"resultsets"`
}