swarm dpa bigfile test

dpabig.go

package main

import (
	"crypto/rand"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"sync"

	sys "golang.org/x/sys/unix"

	"github.com/ethereum/go-ethereum/log"
	"github.com/ethereum/go-ethereum/swarm/storage"
)

const (
	EXTRA_BYTES = 1024 * 1024
)

var (
	datasize int64
	datadir  string
	pagesize int
	keep     bool
)

func init() {
	flag.Int64Var(&datasize, "s", 0, "size of data to write")

	flag.StringVar(&datadir, "d", ".", "dir to write to")

	flag.IntVar(&pagesize, "p", 4096, "size of pages to read back from dpa")

	flag.BoolVar(&keep, "k", false, "do not delete datadir")

	var verbose bool
	flag.BoolVar(&verbose, "v", false, "output debug information")

	var veryverbose bool
	flag.BoolVar(&veryverbose, "vv", false, "output a lot more debug information")

	flag.Parse()

	loglevel := log.LvlWarn
	if veryverbose {
		loglevel = log.LvlTrace
	} else if verbose {
		loglevel = log.LvlDebug
	}
	//log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(loglevel, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(loglevel, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))))
}

func main() {

	if datasize == 0 {
		fmt.Fprintf(os.Stderr, "ERROR: -s required!\n")
		flag.Usage()
		os.Exit(1)
	}

	var stat sys.Statfs_t
	sys.Statfs(datadir, &stat)
	bytesavail := (uint64(stat.Bsize) * stat.Bavail) + EXTRA_BYTES

	if bytesavail < uint64(datasize+EXTRA_BYTES) {
		fmt.Printf("Need %dB, have %dB\n", datasize, bytesavail)
		os.Exit(1)
	}
	dir, err := ioutil.TempDir(datadir, fmt.Sprintf("dpabig-%d-", datasize))
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	if !keep {
		defer os.RemoveAll(dir)
	}

	// in dbcapacity; allow for indices too, 1+64+8+8 bytes per chunk, adds approx 1/40 of volume
	dpa, err := storage.NewLocalDPA(dir, "SHA3", uint64((datasize/storage.CHUNKSIZE)*1.1), 5000)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	dpa.Start()
	defer dpa.Stop()

	swg := &sync.WaitGroup{}
	wwg := &sync.WaitGroup{}

	key, err := dpa.Store(rand.Reader, datasize, swg, wwg)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	log.Debug("stored", "key", key)

	r := dpa.Retrieve(key)
	dpasize, err := r.Size(nil)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	log.Debug("dpa size", "bytes", dpasize)
	dataout := make([]byte, pagesize)
	bytesread := 0
	for {
		c, err := r.Read(dataout)
		bytesread += c
		if err != nil {
			if err != io.EOF {
				log.Error("read fail", "err", err)
			}
			break
		}
	}
	fmt.Printf("bytes read: %d\n", bytesread)
}