compress/uncompress protobuf benchmark test

compress_test.go

package main

import (
	"bytes"
	"compress/flate"
	"compress/gzip"
	"compress/zlib"
	"io/ioutil"
	"math/rand"
	"testing"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/klauspost/compress/zstd"
	"github.com/klauspost/pgzip"
	"github.com/pierrec/lz4"
	pb "zvelo.io/msg/msgpb"
)

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

func BenchmarkProtobufCompress(b *testing.B) {
	msg := &pb.RequestID{RequestId: randSeq(2048)}

	var totalBytes int64
	for i := 0; i < b.N; i++ {
		data, _ := proto.Marshal(msg)
		totalBytes += int64(len(data))
	}
	b.ReportMetric(float64(totalBytes/int64(b.N)), "outlen/op")
}

func BenchmarkProtobufGzipCompress(b *testing.B) {
	msg := &pb.RequestID{RequestId: randSeq(2048)}

	var totalBytes int64
	for i := 0; i < b.N; i++ {
		data, _ := proto.Marshal(msg)
		encoded := gzipCompress(data)
		totalBytes += int64(len(encoded))
	}
	b.ReportMetric(float64(totalBytes/int64(b.N)), "outlen/op")
}

func BenchmarkProtobufZlibCompress(b *testing.B) {
	msg := &pb.RequestID{RequestId: randSeq(2048)}

	var totalBytes int64
	for i := 0; i < b.N; i++ {
		data, _ := proto.Marshal(msg)
		encoded := zlibCompress(data)
		totalBytes += int64(len(encoded))
	}
	b.ReportMetric(float64(totalBytes/int64(b.N)), "outlen/op")
}

func BenchmarkProtobufFlateCompress(b *testing.B) {
	msg := &pb.RequestID{RequestId: randSeq(2048)}

	var totalBytes int64
	for i := 0; i < b.N; i++ {
		data, _ := proto.Marshal(msg)
		encoded := flateCompress(data)
		totalBytes += int64(len(encoded))
	}
	b.ReportMetric(float64(totalBytes/int64(b.N)), "outlen/op")
}

func BenchmarkProtobufPGzipCompress(b *testing.B) {
	msg := &pb.RequestID{RequestId: randSeq(2048)}

	var totalBytes int64
	for i := 0; i < b.N; i++ {
		data, _ := proto.Marshal(msg)
		encoded := pgzipCompress(data)
		totalBytes += int64(len(encoded))
	}
	b.ReportMetric(float64(totalBytes/int64(b.N)), "outlen/op")
}

func BenchmarkProtobufZstdCompress(b *testing.B) {
	msg := &pb.RequestID{RequestId: randSeq(2048)}

	var totalBytes int64
	for i := 0; i < b.N; i++ {
		data, _ := proto.Marshal(msg)
		encoded := zstdCompress(data)
		totalBytes += int64(len(encoded))
	}
	b.ReportMetric(float64(totalBytes/int64(b.N)), "outlen/op")
}

func BenchmarkProtobufLZ4Compress(b *testing.B) {
	msg := &pb.RequestID{RequestId: randSeq(2048)}

	var totalBytes int64
	for i := 0; i < b.N; i++ {
		data, _ := proto.Marshal(msg)
		encoded := lz4Compress(data)
		totalBytes += int64(len(encoded))
	}
	b.ReportMetric(float64(totalBytes/int64(b.N)), "outlen/op")
}

func BenchmarkProtobufUncompress(b *testing.B) {
	data, _ := proto.Marshal(&pb.RequestID{RequestId: randSeq(2048)})

	for i := 0; i < b.N; i++ {
		var msg pb.RequestID
		proto.Unmarshal(data, &msg)
	}
}

func BenchmarkProtobufGzipUncompress(b *testing.B) {
	data, _ := proto.Marshal(&pb.RequestID{RequestId: randSeq(2048)})
	encoded := gzipCompress(data)

	for i := 0; i < b.N; i++ {
		var msg pb.RequestID
		proto.Unmarshal(gzipUncompress(encoded), &msg)
	}
}

func BenchmarkProtobufZlibUncompress(b *testing.B) {
	data, _ := proto.Marshal(&pb.RequestID{RequestId: randSeq(2048)})
	encoded := zlibCompress(data)

	for i := 0; i < b.N; i++ {
		var msg pb.RequestID
		proto.Unmarshal(zlibUncompress(encoded), &msg)
	}
}

func BenchmarkProtobufFlateUncompress(b *testing.B) {
	data, _ := proto.Marshal(&pb.RequestID{RequestId: randSeq(2048)})
	encoded := flateCompress(data)

	for i := 0; i < b.N; i++ {
		var msg pb.RequestID
		proto.Unmarshal(flateUncompress(encoded), &msg)
	}
}

func BenchmarkProtobufPGzipUncompress(b *testing.B) {
	data, _ := proto.Marshal(&pb.RequestID{RequestId: randSeq(2048)})
	encoded := pgzipCompress(data)

	for i := 0; i < b.N; i++ {
		var msg pb.RequestID
		proto.Unmarshal(pgzipUncompress(encoded), &msg)
	}
}

func BenchmarkProtobufZstdUncompress(b *testing.B) {
	data, _ := proto.Marshal(&pb.RequestID{RequestId: randSeq(2048)})
	encoded := zstdCompress(data)

	for i := 0; i < b.N; i++ {
		var msg pb.RequestID
		proto.Unmarshal(zstdUncompress(encoded), &msg)
	}
}

func BenchmarkProtobufLZ4Uncompress(b *testing.B) {
	data, _ := proto.Marshal(&pb.RequestID{RequestId: randSeq(2048)})
	encoded := lz4Compress(data)

	for i := 0; i < b.N; i++ {
		var msg pb.RequestID
		proto.Unmarshal(lz4Uncompress(encoded), &msg)
	}
}

func randSeq(n int) string {
	rand.Seed(time.Now().Unix())

	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func gzipCompress(b []byte) []byte {
	var buf bytes.Buffer
	w := gzip.NewWriter(&buf)
	w.Write(b)
	w.Close()
	return buf.Bytes()
}

func zlibCompress(b []byte) []byte {
	var buf bytes.Buffer
	w := zlib.NewWriter(&buf)
	w.Write(b)
	w.Close()
	return buf.Bytes()
}

func flateCompress(b []byte) []byte {
	var buf bytes.Buffer
	w, _ := flate.NewWriter(&buf, 9)
	w.Write(b)
	w.Close()
	return buf.Bytes()
}

func pgzipCompress(b []byte) []byte {
	var buf bytes.Buffer
	w := pgzip.NewWriter(&buf)
	w.Write(b)
	w.Close()
	return buf.Bytes()
}

func zstdCompress(b []byte) []byte {
	var buf bytes.Buffer
	w, _ := zstd.NewWriter(&buf)
	w.Write(b)
	w.Close()
	return buf.Bytes()
}

func lz4Compress(b []byte) []byte {
	var buf bytes.Buffer
	w := lz4.NewWriter(&buf)
	w.Write(b)
	w.Close()
	return buf.Bytes()
}

func gzipUncompress(b []byte) []byte {
	r, _ := gzip.NewReader(bytes.NewReader(b))
	defer r.Close()

	data, _ := ioutil.ReadAll(r)
	return data
}

func zlibUncompress(b []byte) []byte {
	r, _ := zlib.NewReader(bytes.NewReader(b))
	defer r.Close()

	data, _ := ioutil.ReadAll(r)
	return data
}

func flateUncompress(b []byte) []byte {
	r := flate.NewReader(bytes.NewReader(b))
	defer r.Close()

	data, _ := ioutil.ReadAll(r)
	return data
}

func pgzipUncompress(b []byte) []byte {
	r, _ := pgzip.NewReader(bytes.NewReader(b))
	defer r.Close()

	data, _ := ioutil.ReadAll(r)
	return data
}

func zstdUncompress(b []byte) []byte {
	r, _ := zstd.NewReader(bytes.NewReader(b))
	defer r.Close()

	data, _ := ioutil.ReadAll(r)
	return data
}

func lz4Uncompress(b []byte) []byte {
	r := lz4.NewReader(bytes.NewReader(b))

	data, _ := ioutil.ReadAll(r)
	return data
}

test result:

goos: linux
goarch: amd64
cpu: AMD EPYC 7K62 48-Core Processor
BenchmarkProtobufCompress
BenchmarkProtobufCompress-16           	 1461188	       718.3 ns/op	      2051 outlen/op	    2304 B/op	       1 allocs/op
BenchmarkProtobufGzipCompress
BenchmarkProtobufGzipCompress-16       	    3580	    358159 ns/op	      1521 outlen/op	  820005 B/op	      23 allocs/op
BenchmarkProtobufZlibCompress
BenchmarkProtobufZlibCompress-16       	    3310	    373826 ns/op	      1512 outlen/op	  819947 B/op	      25 allocs/op
BenchmarkProtobufFlateCompress
BenchmarkProtobufFlateCompress-16      	    3246	    362376 ns/op	      1507 outlen/op	  818853 B/op	      21 allocs/op
BenchmarkProtobufPGzipCompress
BenchmarkProtobufPGzipCompress-16      	    2335	    471819 ns/op	      2081 outlen/op	 4749786 B/op	      40 allocs/op
BenchmarkProtobufZstdCompress
BenchmarkProtobufZstdCompress-16       	     302	   3891279 ns/op	      1531 outlen/op	23469697 B/op	      69 allocs/op
BenchmarkProtobufLZ4Compress
BenchmarkProtobufLZ4Compress-16        	    6427	    185641 ns/op	      2070 outlen/op	  734397 B/op	       6 allocs/op
BenchmarkProtobufUncompress
BenchmarkProtobufUncompress-16         	 1323879	       902.6 ns/op	    2072 B/op	       2 allocs/op
BenchmarkProtobufGzipUncompress
BenchmarkProtobufGzipUncompress-16     	   23532	     50418 ns/op	   50251 B/op	      13 allocs/op
BenchmarkProtobufZlibUncompress
BenchmarkProtobufZlibUncompress-16     	   24200	     50565 ns/op	   49666 B/op	      16 allocs/op
BenchmarkProtobufFlateUncompress
BenchmarkProtobufFlateUncompress-16    	   25012	     48682 ns/op	   49545 B/op	      12 allocs/op
BenchmarkProtobufPGzipUncompress
BenchmarkProtobufPGzipUncompress-16    	    2080	    565431 ns/op	 4243244 B/op	      27 allocs/op
BenchmarkProtobufZstdUncompress
BenchmarkProtobufZstdUncompress-16     	   11325	    105108 ns/op	   43578 B/op	     140 allocs/op
BenchmarkProtobufLZ4Uncompress
BenchmarkProtobufLZ4Uncompress-16            807	   1420493 ns/op	 8408930 B/op	      10 allocs/op
PASS