gcs-bench

main.go

package main

import (
	"context"
	"fmt"
	"io"
	"sync"
	"time"

	"cloud.google.com/go/storage"
)

// proving there's no difference between small and large GCS file if you can seek into them
func main() {
	// create gcs client
	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		panic(err)
	}
	ten := client.Bucket("cyriltovena").Object("10mb.bin")
	two := client.Bucket("cyriltovena").Object("2mb.bin")
	write(
		ten.NewWriter(ctx),
		10*1024*1024,
	)
	write(
		two.NewWriter(ctx),
		2*1024*1024,
	)
	readRange(ctx, ten)
	read(ctx, two)
}

func read(ctx context.Context, two *storage.ObjectHandle) {
	start := time.Now()
	readers := make([]*storage.Reader, 0, 5)
	for i := 0; i < 5; i++ {
		r, err := two.NewReader(ctx)
		if err != nil {
			panic(err)
		}
		readers = append(readers, r)
	}
	var wg sync.WaitGroup
	for i, r := range readers {
		wg.Add(1)
		go func(i int, r *storage.Reader) {
			defer wg.Done()
			b, err := io.ReadAll(r)
			if err != nil {
				panic(err)
			}
			fmt.Printf("Read %d bytes from %d\n", len(b), i)
		}(i, r)
	}
	wg.Wait()
	fmt.Printf("Read 10mb in %s\n", time.Since(start))
}

func readRange(ctx context.Context, ten *storage.ObjectHandle) {
	start := time.Now()
	readers := make([]*storage.Reader, 0, 5)
	for i := 0; i < 5; i++ {
		r, err := ten.NewRangeReader(ctx, int64(i*2*1024*1024), int64(2*1024*1024))
		if err != nil {
			panic(err)
		}
		readers = append(readers, r)
	}
	var wg sync.WaitGroup
	for i, r := range readers {
		wg.Add(1)
		go func(i int, r *storage.Reader) {
			defer wg.Done()
			b, err := io.ReadAll(r)
			if err != nil {
				panic(err)
			}
			fmt.Printf("ReadRange %d bytes from %d\n", len(b), i)
		}(i, r)
	}
	wg.Wait()
	fmt.Printf("ReadRange 10mb in %s\n", time.Since(start))
}

func write(writer *storage.Writer, size int) {
	for i := 0; i < size; i++ {
		_, err := writer.Write([]byte("A"))
		if err != nil {
			panic(err)
		}
	}
	if err := writer.Close(); err != nil {
		panic(err)
	}
}

runs.txt

// Run 1
ReadRange 10mb in 386.002583ms
Read 10mb in 431.131638ms

// Run 2
ReadRange 10mb in 358.456829ms
Read 10mb in 332.729691ms

// Run 3
ReadRange 10mb in 391.564784ms
Read 10mb in 362.54284ms

// Run 4
ReadRange 10mb in 321.948104ms
Read 10mb in 476.407386ms