Minimal test case for IPFS concurrent add

readme.md

Run with

go get -d
go build
./createTest.sh
./concurrentadd

concurrentadd.go

// run with ./concurrentadd --dir where/your/testfiles/reside
package main

import (
	"flag"
	"log"
	"os"
	"io/ioutil"
	"github.com/ipfs/go-ipfs-api"
	"path/filepath"
)

var (
	sh *shell.Shell
	ip            = flag.String("ipfsapi", "127.0.0.1:5001", "ipfs api location")
	localDir      = flag.String("dir", "output", "location of local nixipfs path")
)

func init() {
	flag.Parse()
}

func AddToIPFS(c chan bool, s *shell.Shell, file string) {
	f, err := os.Open(file)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	_, err = sh.Add(f)
	if err != nil {
		log.Fatal(err)
	}
	c <- true
	return
}

func listDir(path string) ([]string, error) {
	result := make([]string, 0)
	dir, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	} else {
		for _, entry := range dir {
			if entry.Mode().IsRegular() {
				result = append(result, filepath.Join(path, entry.Name()))
			}
		}
	}
	return result, nil
}

func main() {
	sh = shell.NewShell(*ip)
	files, err := listDir(*localDir)
	if err != nil {
		log.Fatal(err)
	}
	c0 := make(chan bool)
	for _, f := range files {
		go AddToIPFS(c0, sh, f)
	}
	for i := 0; i < len(files); i++ {
		<- c0
	}
}

createTest.sh

#!/usr/bin/env bash

DIR="output"
BS="4K"

mkdir -p $DIR

cd $DIR
for i in `seq 1 1000`; do dd if=/dev/urandom of=test_4k_$i bs=$BS count=1; done
for i in `seq 1 50`; do dd if=/dev/urandom of=test_500k_$i bs=$BS count=125; done
for i in `seq 1 10`; do dd if=/dev/urandom of=test_10M_$i bs=$BS count=2500; done
for i in `seq 1 3`; do dd if=/dev/urandom of=test_100M_$i bs=$BS count=25000; done
for i in `seq 1 2`; do dd if=/dev/urandom of=test_500M_$i bs=$BS count=125000; done
cd -