Local flat JSON db. Ab I/O Heavy monster I wrote in an Evening. So no brain was realy used.

bench.txt

goos: windows
goarch: amd64
pkg: github.com/worldOneo/LilMonk
cpu: Intel(R) Core(TM) i7-8750H CPU @ 2.20GHz
Benchmark_lilmonkdb_composite-12    	     218	   7838672 ns/op	 2308182 B/op	    5140 allocs/op
PASS
ok  	github.com/worldOneo/LilMonk	2.359s

lilmonk.go

package lilmonkdb

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"io/ioutil"
	"math/big"
	"os"
	"path/filepath"
	"time"
)

type LilMonk interface {
	Write(collection, key string, data interface{}) error
	Read(collection, key string, to interface{}) (bool, error)
	Delete(collection, key string) error
	Stop() error
}

type collectionIndex map[string]string

type lilmonkdb struct {
	cIndex     collectionIndex
	cIndexFile string
	directory  string
}

const BinaryFileExtension = "bin"
const CollectionIndexFileName = "index.coll." + BinaryFileExtension

func NewMonk(directory string) (LilMonk, error) {
	absDir, err := filepath.Abs(directory)
	if err != nil {
		return nil, err
	}
	db := &lilmonkdb{}
	err = db.init(absDir)
	if err != nil {
		return nil, err
	}
	return db, err
}

func (l *lilmonkdb) init(directory string) error {
	err := createDirectory(directory)
	if err != nil {
		return err
	}
	l.directory = directory
	return l.initCollectionIndex()
}

func (l *lilmonkdb) Write(collection, key string, data interface{}) error {
	idx, err := l.readCollectionIndex(collection)
	if err != nil {
		return err
	}
	name, ok := idx[key]
	if !ok {
		name = indexName()
		idx[key] = name
		err := l.writeCollectionIndex(collection, idx)
		if err != nil {
			return err
		}
	}

	return writeFileFrom(l.toBinaryFileName(name), data)
}

func (l *lilmonkdb) Delete(collection, key string) error {
	ikey, ok := l.cIndex[collection]
	if !ok {
		return nil
	}
	idx := collectionIndex{}
	err := readFileTo(l.toBinaryFileName(ikey), &idx)
	if err != nil {
		return err
	}
	name, ok := idx[key]
	if !ok {
		return nil
	}
	return os.Remove(l.toBinaryFileName(name))
}

func (l *lilmonkdb) Read(collection, key string, to interface{}) (bool, error) {
	idx, err := l.readCollectionIndex(collection)
	if err != nil {
		return false, err
	}
	name, ok := idx[key]
	if !ok {
		idx[key] = indexName()
		err := l.writeCollectionIndex(collection, idx)
		if err != nil {
			return false, err
		}
	}

	return true, readFileTo(l.toBinaryFileName(name), to)
}

func (l *lilmonkdb) Stop() error {
	return writeFileFrom(l.cIndexFile, &l.cIndex)
}

func (l *lilmonkdb) initCollectionIndex() error {
	l.cIndexFile = filepath.Join(l.directory, CollectionIndexFileName)
	_, err := os.Stat(l.cIndexFile)
	l.cIndex = collectionIndex{}
	if err != nil && os.IsNotExist(err) {
		return nil
	}
	return readFileTo(l.cIndexFile, &l.cIndex)
}

func (l *lilmonkdb) readCollectionIndex(collection string) (collectionIndex, error) {
	key, ok := l.cIndex[collection]

	if !ok {
		idx := collectionIndex{}
		return idx, l.writeCollectionIndex(collection, idx)
	}
	idx := collectionIndex{}
	return idx, readFileTo(l.toBinaryFileName(key), &idx)
}

func (l *lilmonkdb) writeCollectionIndex(collection string, idx collectionIndex) error {
	key, ok := l.cIndex[collection]
	if !ok {
		key = indexName()
		l.cIndex[collection] = key
	}
	return writeFileFrom(l.toBinaryFileName(key), idx)
}

func (l *lilmonkdb) toBinaryFileName(key string) string {
	return filepath.Join(l.directory, key+"."+BinaryFileExtension)
}

func readFileTo(name string, to interface{}) error {
	data, err := readFile(name)
	if err != nil {
		return err
	}
	decoder := json.NewDecoder(bytes.NewReader(data))
	return decoder.Decode(to)
}

func writeFileFrom(name string, from interface{}) error {
	var buff bytes.Buffer
	encoder := json.NewEncoder(&buff)
	err := encoder.Encode(from)
	if err != nil {
		return err
	}
	return writeFile(name, buff.Bytes())
}

func writeFile(name string, data []byte) error {
	file, err := os.OpenFile(name, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0o600)
	if err != nil {
		return err
	}
	defer file.Close()
	writer, err := gzip.NewWriterLevel(file, 4)
	if err != nil {
		return err
	}
	_, err = writer.Write(data)
	writer.Flush()
	writer.Close()
	return err
}

func readFile(name string) ([]byte, error) {
	file, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	reader, err := gzip.NewReader(file)
	if err != nil {
		return nil, err
	}
	return ioutil.ReadAll(reader)
}

func createDirectory(directory string) error {
	_, err := os.Stat(directory)
	if err != nil && os.IsNotExist(err) {
		return os.MkdirAll(directory, 0o600)
	}
	return err
}

func indexName() string {
	return big.NewInt(time.Now().UnixNano()).Text(52)
}

lilmonk_test.go

package lilmonkdb

import (
	"math/rand"
	"testing"
)

type cow struct {
	Name string
	Tag  int
	DNA  string
}

func Test_lilmonkdb(t *testing.T) {
	monk, err := NewMonk("testDir")
	if err != nil {
		t.Fatal(err)
	}
	err = monk.Write("mycollection", "cow1", cow{"Bob", 2, "TTAGGGG"})
	if err != nil {
		t.Fatal(err)
	}
	err = monk.Write("mycollection", "cow2", cow{"Bert", 2, "TGAGGGG"})
	if err != nil {
		t.Fatal(err)
	}

	var myCow cow
	ok, err := monk.Read("mycollection", "cow1", &myCow)
	if err != nil || !ok {
		t.Fatal(err)
	}

	if err = monk.Stop(); err != nil {
		t.Fatal(err)
	}
}

func randomString(n int) []byte {
	var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")

	s := make([]byte, n)
	for i := range s {
		s[i] = letters[rand.Intn(len(letters))]
	}
	return s
}

func generatePayLoad(a, b int) map[string]string {
	payloadmap := make(map[string]string)
	for i := 0; i < a; i++ {
		payloadmap[string(randomString(20))] = string(randomString(b))
	}

	return payloadmap
}

func Benchmark_lilmonkdb_composite(b *testing.B) {
	monk, err := NewMonk("testDir")
	if err != nil {
		b.Fatal(err)
	}
	a := []string{}
	for i := 0; i < b.N; i++ {
		a = append(a, string(randomString(10)))
	}
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		monk.Write("mycollection", a[i], generatePayLoad(100, 40))
	}
	for i := 0; i < b.N; i++ {
		m := map[string]string{}
		monk.Read("mycollection", a[i], &m)
	}
	for i := 0; i < b.N; i++ {
		monk.Delete("mycollection", a[i])
	}
}