Go API Optional Fields

README.md

Handling Optional and Nullable Fields

The API uses the optional package to distinguish between three states for a field in a JSON payload:

1. Omitted: The field is not present in the JSON request body.
2. Set to null: The field is present with an explicit null value (e.g., "description": null). This is used to unset or clear a field's value.
3. Set to a value: The field is present with a non-null value (e.g., "description": "my description" or "description": "").

type Input struct {
	Description optional.String `json:"description"`
}

func main() {
	i := Input{
		Description: optional.NewString("example"),
	}
}

The optional.String type (and similar types for other primitives) provides fields and methods to check for these states:

Omitted(): Returns true if the field was omitted from the request. This is useful in the API layer to check if any updateable fields were provided.

if input.Description.Omitted() {
	// The 'description' field was not in the request body.
}

.Set: A boolean that is true if the field was present in the request, regardless of whether its value was null or not. This is useful in business and data layers to know if a field needs to be processed.

if input.Description.Set {
	// The 'description' field was provided, process the update.
}

.Null: A boolean that is true if the field was present and set to null.

if input.Description.Set && input.Description.Null {
	// The 'description' was explicitly set to null.
}

This allows for PATCH operations where a client can explicitly clear a field by setting it to null, which is different from omitting the field (which means "no change").

The optional package also provides a .ToNullString() method which converts into a sql.NullString type (see database/sql package).

Here is a simple example of what the sql.NullString is useful for:

var name sql.NullString

// Query a row
err := db.QueryRow("SELECT name FROM users WHERE id = ?", 42).Scan(&name)
if err != nil {
    log.Fatal(err)
}

if name.Valid {
    fmt.Println("Name:", name.String)
} else {
    fmt.Println("Name is NULL")
}

optional.go

// Package optional provides data types that can distinguish null and omitted values.
package optional

var nullBytes = []byte("null")

string.go

package optional

import (
	"bytes"
	"database/sql"
	"encoding/json"
)

// String is a JSON serializable string type that is null/omit aware.
type String struct {
	Set   bool
	Null  bool
	Value string
}

// UnmarshalJSON implements json.Unmarshaler.
// If a field has a value of `null`, then Set will be false and Null will be true.
// If a field is omitted from the JSON, then both Set and Null will be false.
func (s *String) UnmarshalJSON(data []byte) error {
	s.Set = true

	if bytes.Equal(data, nullBytes) {
		s.Null = true
		return nil
	}

	return json.Unmarshal(data, &s.Value)
}

// MarshalJSON implements json.Marshaler.
// It will encode to `null` if Null is true.
//
// NOTE: the `omitempty` modifier does not currently work.
func (s String) MarshalJSON() ([]byte, error) {
	if s.Null {
		return nullBytes, nil
	}

	return json.Marshal(s.Value)
}

// Omitted returns true if the value is both unset and not null.
func (s String) Omitted() bool {
	return !s.Set && !s.Null
}

// ToNullString converts the value to a sql.NullString.
//
// sql.NullString.Valid is true when the value is not `Null` and is `Set`
func (s String) ToNullString() sql.NullString {
	return sql.NullString{Valid: !s.Null && s.Set, String: s.Value}
}

// NewString returns a valid, non-null String with the given value.
func NewString(v string) String {
	return String{Value: v, Set: true}
}

// NewStringFromNullString converts a sql.NullString to a String.
func NewStringFromNullString(ns sql.NullString) String {
	return String{Value: ns.String, Set: ns.Valid, Null: !ns.Valid}
}

string_test.go

package optional_test

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"testing"

	"github.com/org/repo/optional"
)

func TestString(t *testing.T) {
	t.Parallel()

	type expect struct {
		value   string
		null    bool
		set     bool
		omitted bool
		json    string
	}

	tests := []struct {
		description string
		in          string
		expect      expect
	}{
		{
			description: "omitted value",
			in:          `{}`,
			expect:      expect{value: "", null: false, set: false, omitted: true, json: `{"name":""}`},
		},
		{
			description: "null value",
			in:          `{"name":null}`,
			expect:      expect{value: "", null: true, set: true, omitted: false, json: `{"name":null}`},
		},
		{
			description: "empty value",
			in:          `{"name":""}`,
			expect:      expect{value: "", null: false, set: true, omitted: false, json: `{"name":""}`},
		},
		{
			description: "non-empty value",
			in:          `{"name":"Bob"}`,
			expect:      expect{value: "Bob", null: false, set: true, omitted: false, json: `{"name":"Bob"}`},
		},
	}

	for _, tc := range tests {
		t.Run(tc.description, func(t *testing.T) {
			p := struct {
				Name optional.String `json:"name,omitempty"`
			}{}

			err := json.Unmarshal([]byte(tc.in), &p)
			if err != nil {
				t.Error("Expected err to be nil:", err)
			}

			if p.Name.Value != tc.expect.value {
				t.Error("Expected Value to be", tc.expect.value, "but got", p.Name.Value)
			}

			if p.Name.Null != tc.expect.null {
				t.Error("Expected IsNull to be", tc.expect.null)
			}

			if p.Name.Set != tc.expect.set {
				t.Error("Expected IsSet to be", tc.expect.null)
			}

			if p.Name.Omitted() != tc.expect.omitted {
				t.Error("Expected Omitted to be", tc.expect.omitted)
			}

			out, err := json.Marshal(p)
			if err != nil {
				t.Error("Expected err to be nil:", err)
			}

			if string(out) != tc.expect.json {
				t.Error("Expected json output to be:", tc.expect.json, ", got:", string(out))
			}
		})
	}
}

func TestNewString(t *testing.T) {
	t.Parallel()

	type expect struct {
		value   string
		null    bool
		set     bool
		omitted bool
	}

	tests := []struct {
		description string
		in          string
		expect      expect
	}{
		{
			description: "empty value",
			in:          "",
			expect:      expect{value: "", null: false, set: true, omitted: false},
		},
		{
			description: "non-empty value",
			in:          "Bob",
			expect:      expect{value: "Bob", null: false, set: true, omitted: false},
		},
	}

	for _, tc := range tests {
		t.Run(tc.description, func(t *testing.T) {
			out := optional.NewString(tc.in)

			if out.Value != tc.expect.value {
				t.Error("Expected Value to be", tc.expect.value, "but got", out.Value)
			}

			if out.Null != tc.expect.null {
				t.Error("Expected IsNull to be", tc.expect.null)
			}

			if out.Set != tc.expect.set {
				t.Error("Expected IsSet to be", tc.expect.null)
			}

			if out.Omitted() != tc.expect.omitted {
				t.Error("Expected Omitted to be", tc.expect.omitted)
			}
		})
	}
}

func TestString_ToNullString(t *testing.T) {
	t.Parallel()

	tests := []struct {
		description string
		in          optional.String
		expect      sql.NullString
	}{
		{
			description: "omitted value",
			in:          optional.String{Value: "", Null: false, Set: false},
			expect:      sql.NullString{Valid: false, String: ""},
		},
		{
			description: "null value",
			in:          optional.String{Value: "", Null: true, Set: false},
			expect:      sql.NullString{Valid: false, String: ""},
		},
		{
			description: "null value set",
			in:          optional.String{Value: "", Null: true, Set: true},
			expect:      sql.NullString{Valid: false, String: ""},
		},
		{
			description: "empty value",
			in:          optional.String{Value: "", Null: false, Set: true},
			expect:      sql.NullString{Valid: true, String: ""},
		},
		{
			description: "non-empty value",
			in:          optional.String{Value: "Bob", Null: false, Set: true},
			expect:      sql.NullString{Valid: true, String: "Bob"},
		},
	}

	for _, tc := range tests {
		t.Run(tc.description, func(t *testing.T) {
			if tc.in.ToNullString().String != tc.expect.String {
				t.Error("Expected String to be", tc.expect.String, "but got", tc.in.ToNullString().String)
			}

			if tc.in.ToNullString().Valid != tc.expect.Valid {
				t.Error("Expected Valid to be", tc.expect.Valid)
			}
		})
	}
}

func ExampleString() {
	type PersonRequest struct {
		Name           optional.String `json:"name"`
		NickName       optional.String `json:"nick_name"`
		HomeTown       optional.String `json:"home_town,omitempty"`
		FavoriteAuthor optional.String `json:"favorite_author"`
	}

	var p PersonRequest

	json.Unmarshal([]byte(`{"name":"Robert", "nick_name":null, "favorite_author":""}`), &p)

	fmt.Printf("Name: Set %v, Null %v, Omitted %v\n", p.Name.Set, p.Name.Null, p.Name.Omitted())
	fmt.Printf("NickName: Set %v, Null %v, Omitted %v\n", p.NickName.Set, p.NickName.Null, p.NickName.Omitted())
	fmt.Printf("HomeTown: Set %v, Null %v, Omitted %v\n", p.HomeTown.Set, p.HomeTown.Null, p.HomeTown.Omitted())
	fmt.Printf("FavoriteAuthor: Set %v, Null %v, Omitted %v\n", p.FavoriteAuthor.Set, p.FavoriteAuthor.Null, p.FavoriteAuthor.Omitted())

	// Output:
	// Name: Set true, Null false, Omitted false
	// NickName: Set true, Null true, Omitted false
	// HomeTown: Set false, Null false, Omitted true
	// FavoriteAuthor: Set true, Null false, Omitted false
}

func ExampleString_MarshalJSON() {
	type PersonRequest struct {
		Name           optional.String `json:"name"`
		NickName       optional.String `json:"nick_name"`
		HomeTown       optional.String `json:"home_town,omitempty"`
		FavoriteAuthor optional.String `json:"favorite_author"`
	}

	p := PersonRequest{
		Name:           optional.NewString("Robert"),
		NickName:       optional.String{Value: "", Null: true, Set: true},
		FavoriteAuthor: optional.NewString(""),
	}

	data, err := json.Marshal(&p)

	fmt.Printf("Data is: %s\n", data)
	fmt.Printf("Error is: %v\n", err)

	// Output:
	// Data is: {"name":"Robert","nick_name":null,"home_town":"","favorite_author":""}
	// Error is: <nil>
}

time.go

package optional

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"time"
)

// Time wraps a time.Time and implements the json.Marshaler interface
// to marhsal the inner time value to "null" if it's zero.
type Time struct {
	Value time.Time
	Set   bool
	Null  bool
}

// UnmarshalJSON implements json.Unmarshaler.
// If a field has a value of `null`, then Set will be false and Null will be true.
// If a field is omitted from the JSON, then both Set and Null will be false.
func (t *Time) UnmarshalJSON(data []byte) error {
	t.Set = true

	if bytes.Equal(data, nullBytes) {
		t.Null = true
		return nil
	}

	return json.Unmarshal(data, &t.Value)
}

// MarshalJSON implements json.Marshaler.
// It will encode to `null` if Null is true or the time value is zero.
//
// NOTE: the `omitempty` modifier does not currently work.
func (t Time) MarshalJSON() ([]byte, error) {
	if t.Null || t.Set && t.Value.IsZero() {
		return nullBytes, nil
	}

	return t.Value.MarshalJSON()
}

// NewTimeFromNullTime converts a sql.NullTime to a NullTime.
func NewTimeFromNullTime(ns sql.NullTime) Time {
	return Time{
		Value: ns.Time,
		Set:   ns.Valid,
		Null:  !ns.Valid,
	}
}

time_test.go

package optional_test

import (
	"encoding/json"
	"testing"
	"time"

	"github.com/org/repo/optional"
)

func TestTime(t *testing.T) {
	t.Parallel()

	type expect struct {
		value   time.Time
		null    bool
		set     bool
		omitted bool
		json    string
	}

	timestamp, err := time.Parse(time.RFC3339, "2021-03-08T22:51:54Z")
	if err != nil {
		t.Fatalf("Failed to create timestamp time for testing: %v", err)
	}

	tests := []struct {
		description string
		in          string
		expect      expect
	}{
		{
			description: "omitted value",
			in:          `{}`,
			expect:      expect{value: time.Time{}, null: false, set: false, omitted: true, json: `{"timestamp":"0001-01-01T00:00:00Z"}`},
		},
		{
			description: "null value",
			in:          `{"timestamp":null}`,
			expect:      expect{value: time.Time{}, null: true, set: true, omitted: false, json: `{"timestamp":null}`},
		},
		{
			description: "empty value",
			in:          `{"timestamp":"0001-01-01T00:00:00Z"}`,
			expect:      expect{value: time.Time{}, null: false, set: true, omitted: false, json: `{"timestamp":null}`},
		},
		{
			description: "non-empty value",
			in:          `{"timestamp":"2021-03-08T22:51:54Z"}`,
			expect:      expect{value: timestamp, null: false, set: true, omitted: false, json: `{"timestamp":"2021-03-08T22:51:54Z"}`},
		},
	}

	for _, tc := range tests {
		t.Run(tc.description, func(t *testing.T) {
			p := struct {
				Timestamp optional.Time `json:"timestamp,omitempty"`
			}{}

			err := json.Unmarshal([]byte(tc.in), &p)
			if err != nil {
				t.Error("Expected err to be nil:", err)
			}

			if p.Timestamp.Value != tc.expect.value {
				t.Error("Expected Value to be", tc.expect.value, "but got", p.Timestamp.Value)
			}

			if p.Timestamp.Null != tc.expect.null {
				t.Error("Expected IsNull to be", tc.expect.null)
			}

			if p.Timestamp.Set != tc.expect.set {
				t.Error("Expected IsSet to be", tc.expect.null)
			}

			out, err := json.Marshal(p)
			if err != nil {
				t.Error("Expected err to be nil:", err)
			}

			if string(out) != tc.expect.json {
				t.Error("Expected json output to be:", tc.expect.json, ", got:", string(out))
			}
		})
	}
}