Skip to content

Instantly share code, notes, and snippets.

@jpfluger

jpfluger/json-key.go

Last active October 17, 2020 17:55
Show Gist options
  • Save jpfluger/046354e64ff9eba1ebd7697adc2f8798 to your computer and use it in GitHub Desktop.
Save jpfluger/046354e64ff9eba1ebd7697adc2f8798 to your computer and use it in GitHub Desktop.
Download ZIP
Creating all literal GJSON Paths
Raw
json-key.go
package lib
import (
"fmt"
"strings"
)
type JsonKey string
type JsonKeys []JsonKey
func (jk JsonKey) IsEmpty() bool {
rt := strings.TrimSpace(string(jk))
return rt == ""
}
func (jk JsonKey) TrimSpace() JsonKey {
key := strings.TrimSpace(string(jk))
return JsonKey(key)
}
func (jk JsonKey) String() string {
return strings.TrimSpace(string(jk))
}
func (jk JsonKey) IsRoot() bool {
return !strings.Contains(jk.String(), ".")
}
func (jk JsonKey) GetPathLeaf() string {
if jk.IsRoot() {
return jk.String()
}
ss := jk.GetPathParts()
return ss[len(ss) - 1]
}
func (jk JsonKey) GetPathParts() []string {
ss := strings.Split(jk.String(), ".")
return ss
}
func (jk JsonKey) GetPathParent() string {
if jk.IsRoot() {
return ""
}
return jk.String()[0:strings.LastIndex(jk.String(), ".")]
}
func (jk *JsonKey) Add(target JsonKey) JsonKey {
if target.IsEmpty() {
return ""
}
if jk.IsEmpty() {
*jk = target
} else {
*jk = JsonKey(fmt.Sprintf("%s.%s", jk.String(), target.String()))
}
return *jk
}
func (jk JsonKey) CopyPlusAdd(target JsonKey) JsonKey {
if target.IsEmpty() {
return ""
}
jkNew := jk
if jkNew.IsEmpty() {
jkNew = target
} else {
jkNew = JsonKey(fmt.Sprintf("%s.%s", jk.String(), target.String()))
}
return jkNew
}
func (jk JsonKey) CopyPlusAddInt(target int) JsonKey {
if target < 0 {
target = 0
}
jkNew := jk
if jkNew.IsEmpty() {
jkNew = JsonKey(fmt.Sprintf("%d", target))
} else {
jkNew = JsonKey(fmt.Sprintf("%s.%d", jk.String(), target))
}
return jkNew
}
Raw
walk-json.go
package lib
import (
"bytes"
"encoding/json"
"fmt"
)
type WalkJsonType int
const (
// call just after the object is unmarshalled - with any error
WALKJSONTYPE_OBJECT_PRE WalkJsonType = iota
// call after object processing - after any inside elements are unmarshalled
WALKJSONTYPE_OBJECT_POST
// call just after the array is unmarshalled - with any error
WALKJSONTYPE_ARRAY_PRE
// call after array processing - after any inside elements are unmarshalled
WALKJSONTYPE_ARRAY_POST
WALKJSONTYPE_INT
WALKJSONTYPE_FLOAT
WALKJSONTYPE_STRING
WALKJSONTYPE_BOOL
WALKJSONTYPE_NIL
)
type WalkJsonDecisionType int
const (
// Continue normal processing of the walk
WALKJSON_DECISIONTYPE_CONTINUE WalkJsonDecisionType = iota
// Exit the walk
WALKJSON_DECISIONTYPE_EXIT
// Skip this section of the walk. Applies to objects and arrays only.
WALKJSON_DECISIONTYPE_SKIP
)
type RawMap map[string]json.RawMessage
type RawArray []json.RawMessage
type FNWalkJsonCallback func(walkJsonType WalkJsonType, level int, fldName JsonKey, val interface{}) (WalkJsonDecisionType, error)
func WalkJson(b []byte, fnCallback FNWalkJsonCallback) error {
var tmpJ json.RawMessage
if err := json.Unmarshal(b, &tmpJ); err != nil {
return fmt.Errorf("unable to unmarshal json; %v", err)
}
return WalkJsonRawMessage(tmpJ, fnCallback)
}
// Walk the raw JSON message allowing the passed-in callback function to control whether the walk continues, skips or returns
// The returned "error" should be treated as a "panic" or "fatal" error to the walk.
// Actual field-level validations should be handled by the callback function (eg a map containing errors = map[JsonKey]error)
func WalkJsonRawMessage(raw json.RawMessage, fnCallback FNWalkJsonCallback) error {
if fnCallback == nil {
return fmt.Errorf("fnCallback is nil")
}
_, err := walkJson(raw, 0, "", fnCallback)
return err
}
func walkJson(raw json.RawMessage, level int, fldName JsonKey, fnCallback FNWalkJsonCallback) (WalkJsonDecisionType, error) {
fldName = fldName.TrimSpace()
if level < 0 {
level = 0
}
// Drew inspiration from https://github.com/laszlothewiz/golang-snippets-examples/blob/master/walk-JSON-tree.go
// to filter by the byte value and parse on the fly instead of converting directly to map[string]interface{}.
if raw[0] == byte(123) { // 123 is ascii byte value for `{`
// Pre object processing suggestions of use:
// 1. initialize a new object, then fill in as the walk continues
// 2. initialize error handling map, then fill as sub-elements are processed
if decisionType, err := fnCallback(WALKJSONTYPE_OBJECT_PRE, level, fldName, raw); err != nil || decisionType != WALKJSON_DECISIONTYPE_CONTINUE {
return decisionType, err
}
var cont RawMap
if err := json.Unmarshal(raw, &cont); err != nil {
return WALKJSON_DECISIONTYPE_EXIT, fmt.Errorf("failed to unmarshal json object (fldName='%s'); %v", fldName, err)
}
for key, v := range cont {
if decisionType, err := walkJson(v, level + 1, fldName.CopyPlusAdd(JsonKey(key)), fnCallback); err != nil || decisionType == WALKJSON_DECISIONTYPE_EXIT {
return decisionType, err
}
}
// Post object processing suggestions of use:
// 1. trigger custom actions like "save"
// 2. trigger "validate" action, esp those that are dependent upon other fields in the object
// 3. at the root level, a "validate" action could validate against fields in the entire object rather than just a sub-level object
return fnCallback(WALKJSONTYPE_OBJECT_POST, level, fldName, raw)
} else if raw[0] == byte(91) { // 91 is ascii byte value `[`
// Pre array processing suggestions of use:
// 1. for validation purposes, set the expected type for all array elements
if decisionType, err := fnCallback(WALKJSONTYPE_ARRAY_PRE, level, fldName, raw); err != nil || decisionType != WALKJSON_DECISIONTYPE_CONTINUE {
return decisionType, err
}
var cont RawArray
if err := json.Unmarshal(raw, &cont); err != nil {
return WALKJSON_DECISIONTYPE_EXIT, fmt.Errorf("failed to unmarshal json array (fldName='%s'); %v", fldName, err)
}
for ii, v := range cont {
if decisionType, err := walkJson(v, level + 1, fldName.CopyPlusAddInt(ii), fnCallback); err != nil || decisionType == WALKJSON_DECISIONTYPE_EXIT {
return decisionType, err
}
}
// Post array processing suggestions of use:
// 1. trigger custom actions like "save"
// 2. trigger "validate" action, esp those that are dependent upon other values in the array
return fnCallback(WALKJSONTYPE_OBJECT_POST, level, fldName, raw)
} else {
var val interface{}
// https://stackoverflow.com/questions/53422587/need-to-parse-integers-in-json-as-integers-not-floats
// https://play.golang.org/p/W4fKXZTkNG
dec := json.NewDecoder(bytes.NewReader(raw))
dec.UseNumber()
err := dec.Decode(&val)
if err != nil {
return WALKJSON_DECISIONTYPE_EXIT, fmt.Errorf("failed to decode json value (fldName='%s'); %v", fldName, err)
}
switch val.(type) {
case json.Number:
if n, err := val.(json.Number).Int64(); err == nil {
return fnCallback(WALKJSONTYPE_INT, level, fldName, n)
} else if f, err := val.(json.Number).Float64(); err == nil {
return fnCallback(WALKJSONTYPE_FLOAT, level, fldName,f)
} else {
// unknown... assume a string
return fnCallback(WALKJSONTYPE_STRING, level, fldName, fmt.Sprintf("%v", val))
}
case string:
return fnCallback(WALKJSONTYPE_STRING, level, fldName, val)
case bool:
return fnCallback(WALKJSONTYPE_BOOL, level, fldName, val)
case nil:
return fnCallback(WALKJSONTYPE_NIL, level, fldName, nil)
default:
// unknown... assume a string
return fnCallback(WALKJSONTYPE_STRING, level, fldName, fmt.Sprintf("%v", val))
}
}
}
Raw
walk-json_test.go
package lib
import (
"github.com/stretchr/testify/assert"
"testing"
)
var jsonTestWalker = []byte(`{
"fld_string_empty":"",
"fld_string_value":"value",
"fld_string_nil":null,
"fld_int_0":0,
"fld_int_pos":99,
"fld_int_neg":-99,
"fld_int_nil":null,
"fld_float_0":0,
"fld_float_0_0":0.0,
"fld_float_pos":2.5,
"fld_float_pos2":500,
"fld_float_neg":-2.5,
"fld_float_nil":null,
"fld_bool_true":true,
"fld_bool_false":false,
"fld_bool_nil":null,
"fld_obj": {
"z1":"o0",
"z2":"o1"
},
"fld_arr":["a0","a1","a2"],
"deeply": {
"nested": {
"object": {
"fld_string_empty":"",
"fld_string_value":"value",
"fld_string_nil":null,
"fld_int_0":0,
"fld_int_pos":99,
"fld_int_neg":-99,
"fld_int_nil":null,
"fld_float_0":0,
"fld_float_0_0":0.0,
"fld_float_pos":2.5,
"fld_float_pos2":500,
"fld_float_neg":-2.5,
"fld_float_nil":null,
"fld_bool_true":true,
"fld_bool_false":false,
"fld_bool_nil":null,
"fld_obj": {
"z1":"o0",
"z2":"o1"
},
"fld_arr":["a0","a1","a2"]
}
}
}
}`)
type MatchJsonValue struct {
Type WalkJsonType
Value interface{}
}
func TestWalkJson(t *testing.T) {
keys := JsonKeys{}
matches := map[JsonKey]MatchJsonValue{}
fnHandler := func(walkJsonType WalkJsonType, level int, fldName JsonKey, val interface{}) (WalkJsonDecisionType, error) {
switch walkJsonType {
case WALKJSONTYPE_OBJECT_POST, WALKJSONTYPE_ARRAY_POST:
break
default:
if !fldName.IsEmpty() {
keys = append(keys, fldName)
matches[fldName] = MatchJsonValue{Type: walkJsonType, Value: val}
}
}
return WALKJSON_DECISIONTYPE_CONTINUE, nil
}
if err := WalkJson(jsonTestWalker, fnHandler); err != nil {
t.Fatal(err)
}
assert.Equal(t, len(keys), len(matches))
for _, key := range keys {
// fmt.Println(key)
if key.IsEmpty() {
t.Fatalf("key is empty (key='%s')", key)
}
}
// Expected keys
expected := map[JsonKey]MatchJsonValue{
"fld_string_empty": {Type: WALKJSONTYPE_STRING, Value: ""},
"fld_string_value": {Type: WALKJSONTYPE_STRING, Value: "value"},
"fld_string_nil": {Type: WALKJSONTYPE_NIL, Value: nil},
"fld_int_0": {Type: WALKJSONTYPE_INT, Value: int64(0)},
"fld_int_pos": {Type: WALKJSONTYPE_INT, Value: int64(99)},
"fld_int_neg": {Type: WALKJSONTYPE_INT, Value: int64(-99)},
"fld_int_nil": {Type: WALKJSONTYPE_NIL, Value: nil},
"fld_float_0": {Type: WALKJSONTYPE_FLOAT, Value: float64(0)},
"fld_float_0_0": {Type: WALKJSONTYPE_FLOAT, Value: float64(0.0)},
"fld_float_pos": {Type: WALKJSONTYPE_FLOAT, Value: float64(2.5)},
"fld_float_pos2": {Type: WALKJSONTYPE_FLOAT, Value: float64(500)},
"fld_float_neg": {Type: WALKJSONTYPE_FLOAT, Value: float64(-2.5)},
"fld_float_nil": {Type: WALKJSONTYPE_NIL, Value: nil},
"fld_bool_true": {Type: WALKJSONTYPE_BOOL, Value: true},
"fld_bool_false": {Type: WALKJSONTYPE_BOOL, Value: false},
"fld_bool_nil": {Type: WALKJSONTYPE_NIL, Value: nil},
"fld_obj": {Type: WALKJSONTYPE_OBJECT_PRE, Value: nil},
"fld_obj.z1": {Type: WALKJSONTYPE_STRING, Value: "o0"},
"fld_obj.z2": {Type: WALKJSONTYPE_STRING, Value: "o1"},
"fld_arr": {Type: WALKJSONTYPE_ARRAY_PRE, Value: nil},
"fld_arr.0": {Type: WALKJSONTYPE_STRING, Value: "a0"},
"fld_arr.1": {Type: WALKJSONTYPE_STRING, Value: "a1"},
"fld_arr.2": {Type: WALKJSONTYPE_STRING, Value: "a2"},
"deeply": {Type: WALKJSONTYPE_OBJECT_PRE, Value: nil},
"deeply.nested": {Type: WALKJSONTYPE_OBJECT_PRE, Value: nil},
"deeply.nested.object": {Type: WALKJSONTYPE_OBJECT_PRE, Value: nil},
"deeply.nested.object.fld_string_empty": {Type: WALKJSONTYPE_STRING, Value: ""},
"deeply.nested.object.fld_string_value": {Type: WALKJSONTYPE_STRING, Value: "value"},
"deeply.nested.object.fld_string_nil": {Type: WALKJSONTYPE_NIL, Value: nil},
"deeply.nested.object.fld_int_0": {Type: WALKJSONTYPE_INT, Value: int64(0)},
"deeply.nested.object.fld_int_pos": {Type: WALKJSONTYPE_INT, Value: int64(99)},
"deeply.nested.object.fld_int_neg": {Type: WALKJSONTYPE_INT, Value: int64(-99)},
"deeply.nested.object.fld_int_nil": {Type: WALKJSONTYPE_NIL, Value: nil},
"deeply.nested.object.fld_float_0": {Type: WALKJSONTYPE_FLOAT, Value: float64(0)},
"deeply.nested.object.fld_float_0_0": {Type: WALKJSONTYPE_FLOAT, Value: float64(0.0)},
"deeply.nested.object.fld_float_pos": {Type: WALKJSONTYPE_FLOAT, Value: float64(2.5)},
"deeply.nested.object.fld_float_pos2": {Type: WALKJSONTYPE_FLOAT, Value: float64(500)},
"deeply.nested.object.fld_float_neg": {Type: WALKJSONTYPE_FLOAT, Value: float64(-2.5)},
"deeply.nested.object.fld_float_nil": {Type: WALKJSONTYPE_NIL, Value: nil},
"deeply.nested.object.fld_bool_true": {Type: WALKJSONTYPE_BOOL, Value: true},
"deeply.nested.object.fld_bool_false": {Type: WALKJSONTYPE_BOOL, Value: false},
"deeply.nested.object.fld_bool_nil": {Type: WALKJSONTYPE_NIL, Value: nil},
"deeply.nested.object.fld_obj": {Type: WALKJSONTYPE_OBJECT_PRE, Value: nil},
"deeply.nested.object.fld_obj.z1": {Type: WALKJSONTYPE_STRING, Value: "o0"},
"deeply.nested.object.fld_obj.z2": {Type: WALKJSONTYPE_STRING, Value: "o1"},
"deeply.nested.object.fld_arr": {Type: WALKJSONTYPE_ARRAY_PRE, Value: nil},
"deeply.nested.object.fld_arr.0": {Type: WALKJSONTYPE_STRING, Value: "a0"},
"deeply.nested.object.fld_arr.1": {Type: WALKJSONTYPE_STRING, Value: "a1"},
"deeply.nested.object.fld_arr.2": {Type: WALKJSONTYPE_STRING, Value: "a2"},
}
assert.Equal(t, len(expected), len(matches))
for key, expect := range expected {
match, ok := matches[key]
if !ok {
t.Fatalf("missing key '%s' in matches", key)
}
switch expect.Type {
case WALKJSONTYPE_STRING, WALKJSONTYPE_INT, WALKJSONTYPE_BOOL:
assert.Equal(t, expect.Type, match.Type)
assert.Equal(t, expect.Value, match.Value)
case WALKJSONTYPE_FLOAT:
// when number is an integer, if the expected type is float, then must convert
if expect.Type == WALKJSONTYPE_FLOAT && match.Type != WALKJSONTYPE_FLOAT {
switch val := match.Value.(type) {
case float64:
match.Type = WALKJSONTYPE_FLOAT
match.Value = val
case float32:
match.Type = WALKJSONTYPE_FLOAT
match.Value = float64(val)
case int64:
match.Type = WALKJSONTYPE_FLOAT
match.Value = float64(val)
default:
t.Fatalf("unknown val of incompatible type (val='%v')", val)
}
}
assert.Equal(t, expect.Type, match.Type)
assert.Equal(t, expect.Value, match.Value)
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment