Skip to content

Instantly share code, notes, and snippets.

@korjavin
Created February 21, 2017 08:59
Show Gist options
  • Save korjavin/afe35d11165bbf916e279158c55e7537 to your computer and use it in GitHub Desktop.
Save korjavin/afe35d11165bbf916e279158c55e7537 to your computer and use it in GitHub Desktop.
package main
import (
"encoding/json"
"github.com/tidwall/buntdb"
"github.com/tidwall/gjson"
"log"
"strconv"
)
type Order struct {
Oid int
Statuses []Orderstatus
}
type Orderstatus struct {
Sid int
Description string
}
func main() {
o1 := Order{Oid: 1, Statuses: []Orderstatus{Orderstatus{Sid: 1, Description: "Isn't relevant"}, Orderstatus{Sid: 2, Description: "Skip that"}}}
o2 := Order{Oid: 2, Statuses: []Orderstatus{Orderstatus{Sid: 1, Description: "Isn't relevant"}, Orderstatus{Sid: 6, Description: "This one"}}}
db, _ := buntdb.Open(":memory:")
buf1, _ := json.Marshal(o1)
buf2, _ := json.Marshal(o2)
db.Update(func(tx *buntdb.Tx) error {
_, _, err := tx.Set(strconv.Itoa(o1.Oid), string(buf1), nil)
_, _, err = tx.Set(strconv.Itoa(o2.Oid), string(buf2), nil)
return err
})
db.Update(func(tx *buntdb.Tx) error {
err := tx.CreateIndex("sid", "*", buntdb.IndexJSON("Statuses.Sid"))
return err
})
db.View(func(tx *buntdb.Tx) error {
tx.AscendGreaterOrEqual("sid", `{"Statuses.Sid":6}`, func(key, value string) bool {
var order Order
json.Unmarshal([]byte(value), &order)
name := gjson.Get(value, `Statuses.#[Sid=6].Description`)
log.Printf("value=%s,key=%s, sid=%s\n", value, key, name)
return true
})
return nil
})
}
@tidwall
Copy link

tidwall commented Feb 21, 2017

AFAIK it's not possible to index on a JSON array efficiently, such as the Statuses element.

In your code the index is looking for an object at the path Statuses.Sid, but your JSON schema does not contain that path.

o1 := Order{Oid: 1, Statuses: []Orderstatus{Orderstatus{Sid: 1, Description: "Isn't relevant"}, Orderstatus{Sid: 2, Description: "Skip that"}}}
o2 := Order{Oid: 2, Statuses: []Orderstatus{Orderstatus{Sid: 1, Description: "Isn't relevant"}, Orderstatus{Sid: 6, Description: "This one"}}}
buf1, _ := json.Marshal(o1)
buf2, _ := json.Marshal(o2)

buf1 and buf2 become:

{"Oid":1,"Statuses":[{"Sid":1,"Description":"Isn't relevant"},{"Sid":2,"Description":"Skip that"}]}
{"Oid":2,"Statuses":[{"Sid":1,"Description":"Isn't relevant"},{"Sid":6,"Description":"This one"}]}

When you enter one of the payloads into http://tidwall.com/gjson-play and use Statuses.Sid as the path you will see that result is null.


So you have a couple options.

Do a full scan

If you don't have a ton of items, you can scan the entire dataset like such:

package main

import (
	"encoding/json"
	"log"
	"strconv"

	"github.com/tidwall/buntdb"
	"github.com/tidwall/gjson"
)

type Order struct {
	Oid      int
	Statuses []Orderstatus
}
type Orderstatus struct {
	Sid         int
	Description string
}

func main() {
	o1 := Order{Oid: 1, Statuses: []Orderstatus{Orderstatus{Sid: 1, Description: "Isn't relevant"}, Orderstatus{Sid: 2, Description: "Skip that"}}}
	o2 := Order{Oid: 2, Statuses: []Orderstatus{Orderstatus{Sid: 1, Description: "Isn't relevant"}, Orderstatus{Sid: 6, Description: "This one"}}}

	db, _ := buntdb.Open(":memory:")
	buf1, _ := json.Marshal(o1)
	buf2, _ := json.Marshal(o2)

	db.Update(func(tx *buntdb.Tx) error {
		_, _, err := tx.Set(strconv.Itoa(o1.Oid), string(buf1), nil)
		_, _, err = tx.Set(strconv.Itoa(o2.Oid), string(buf2), nil)
		return err
	})

	db.View(func(tx *buntdb.Tx) error {
		tx.Ascend("", func(key, value string) bool {
			gjson.Get(value, "Statuses").ForEach(func(_, result gjson.Result) bool {
				sid := result.Get("Sid").Int()
				if sid == 6 {
					name := result.Get("Description")
					log.Printf("value=%s,key=%s, sid=%s\n", value, key, name)
				}
				return true
			})
			return true
		})
		return nil
	})
}

Create a many to many index

if you do have a lot of items then I recommend that you insert each Sid/Oid pair individually.

It's more code an operations, but it will be very fast when querying.

tx.Set("oid:1", `{"Oid":1,"Statuses":[{"Sid":1,"Description":"Isn't relevant"},{"Sid":2,"Description":"Skip that"}]}`, nil)
tx.Set("oid:2", `{"Oid":2,"Statuses":[{"Sid":1,"Description":"Isn't relevant"},{"Sid":6,"Description":"This one"}]}`, nil)
tx.Set("oid_sid:1:1",`{"Oid":1,"Sid":1}`,nil)
tx.Set("oid_sid:1:2",`{"Oid":1,"Sid":2}`,nil)
tx.Set("oid_sid:2:1",`{"Oid":2,"Sid":1}`,nil)
tx.Set("oid_sid:2:6",`{"Oid":2,"Sid":6}`,nil)

Create an index only on the oid_sid: keys, known that each will have exactly one Oid and one Sid.

tx.CreateIndex("oid_sids", "oid_sid:*", buntdb.IndexJSON("Sid"))

Finally search for the Sid. This query will find all Oids that contain an Orderstatus with a Sid of 6.

db.View(func(tx *buntdb.Tx) error {
	tx.AscendGreaterOrEqual("oid_sids", `{"Sid":6}`, func(key, value string) bool {
		if tx.Get("Sid").Int() != 6{
			return false
		}
		value, _ =tx.Get("oid:"+gjson.Get(value, "Oid").String()) // value is now the parent order
		name := gjson.Get(value, `Statuses.#[Sid=6].Description`)
		log.Printf("value=%s,key=%s, sid=%s\n", value, key, name)
		return true
	})
})

This is what I came up with from a cursory look and I haven't tested the code.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment