Using AST's in Go together with git submodules to Detect Incoming Structs

ast_expectations.go

package future

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"sort"
	"strings"
	"testing"
)

type missing struct {
	shouldMiss bool
	msg        string
}

func (hm *missing) handleMissing(isMissing bool) error {
	if hm.shouldMiss != isMissing {
		return fmt.Errorf(hm.msg)
	}
	return nil
}

type expectation struct {
	File        string
	MissingFile missing

	Name          string
	MissingStruct missing

	Fields        []string
	MissingFields missing
}

func getProblematicExpectations(t *testing.T, expectations []expectation) (problems [][]error) {
	problems = [][]error{}

	for i, expected := range expectations {
		fmt.Printf(`
____________________________________________
%d. Expectations for file %s
Expecting: %#v`, i+1, expected.File, expected)

		prob := []error{}

		fset := token.NewFileSet()
		root, err := parser.ParseFile(fset, expected.File, nil, parser.ParseComments)

		missingFile := false
		if err != nil {
			fmt.Printf("\nHeads up! ParseFile failed on [%v]", err)
			missingFile = true
		}

		fileExpectationError := expected.MissingFile.handleMissing(missingFile)
		if fileExpectationError != nil {
			prob = append(prob, fileExpectationError)
		}

		if missingFile {
			problems = append(problems, prob)
			continue
		}

		fileStructs := getFileStructs(root)
		fmt.Printf("\nStructs found in %s: %s", expected.File, fileStructs)

		foundStruct, ok := fileStructs[expected.Name]
		structExpectationError := expected.MissingStruct.handleMissing(!ok)
		if structExpectationError != nil {
			prob = append(prob, structExpectationError)
		}

		if !ok {
			problems = append(problems, prob)
			continue
		}

		missingSomeField := false
		for _, field := range expected.Fields {
			_, ok := foundStruct[field]
			if !ok {
				missingSomeField = true
				break
			}
		}
		fieldsExpectationError := expected.MissingFields.handleMissing(missingSomeField)
		if fieldsExpectationError != nil {
			prob = append(prob, fieldsExpectationError)
		}
		problems = append(problems, prob)
	}

	fmt.Printf("\nAll Problems: %+v", problems)

	return
}

func getTopLevelTypeNodes(root *ast.File) (typeSpecs []*ast.TypeSpec) {
	for _, decl := range root.Decls {
		if decl == nil {
			continue
		}
		gDecl, ok := decl.(*ast.GenDecl)
		if !ok || gDecl == nil {
			continue
		}
		for _, spec := range gDecl.Specs {
			tSpec, ok := spec.(*ast.TypeSpec)
			if !ok || tSpec == nil {
				continue
			}
			typeSpecs = append(typeSpecs, tSpec)
		}
	}
	return
}

// first elmt of stct is the struct's name, the rest are fields
func getStructFieldNames(tSpec *ast.TypeSpec) (stct []string) {
	if tSpec == nil || tSpec.Type == nil {
		return
	}
	stct = append(stct, tSpec.Name.Name)
	sType, ok := tSpec.Type.(*ast.StructType)
	if !ok {
		return
	}
	if sType.Fields == nil || sType.Fields.List == nil {
		return
	}
	for _, field := range sType.Fields.List {
		if field == nil || field.Names == nil {
			continue
		}
		for _, name := range field.Names {
			stct = append(stct, name.Name)
		}
	}
	return
}

type fStructs map[string]map[string]bool

func (fs fStructs) String() string {
	parts := make([]string, len(fs))
	for stct, fset := range fs {
		fields := []string{}
		for field := range fset {
			fields = append(fields, field)
		}
		sort.Strings(fields)
		parts = append(parts, fmt.Sprintf("%s: %s", stct, fields))
	}
	return strings.Join(parts, "\n")
}

// returns map from struct's name, to its set of field names
func getFileStructs(root *ast.File) fStructs {
	fileStructs := map[string]map[string]bool{}
	for _, tSpec := range getTopLevelTypeNodes(root) {
		structInfo := getStructFieldNames(tSpec)
		if len(structInfo) > 0 {
			fields := map[string]bool{}
			for _, field := range structInfo[1:] {
				fields[field] = true
			}
			fileStructs[structInfo[0]] = fields
		}
	}
	return fileStructs
}

ast_expectations_test.go

package future

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

var expectations = []expectation{
	{
		// we should NOT see the file third_party/go-algorand/ledger/ledgercore/accountdata.go:
		"../third_party/go-algorand/ledger/ledgercore/accountdata.go",
		missing{shouldMiss: true, msg: "\n!!!path ledger/ledgercore/accountdata.go detected ==> unlimited assets has arrived\n"},

		// AND struct AccountBaseData should not be available either:
		"AccountBaseData",
		missing{shouldMiss: true, msg: "\n!!!struct AccountBaseData is detected ==> unlimited assets has arrived\n"},

		// AND finally, AccountBaseData.TotalAssets also should not exist:
		[]string{"TotalAssets"},
		missing{shouldMiss: true, msg: "\n!!!field TotalAssets is detected ==> unlimited assets has arrived\n"},
	},
}

func TestTheFuture(t *testing.T) {
	problems := getProblematicExpectations(t, expectations)

	for i, prob := range problems {
		for _, err := range prob {
			assert.NoError(t, err, "error in expectation #%d (%s):\n%v", i+1, expectations[i].File, err)
		}
	}
}

fixture.go

package future

type msg struct {
	Subject string
	Body    []byte
}

fixture_test.go

package future

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var fixtureExpectations = []expectation{
	// missing DNE.go and happy about that
	{
		"./DNE.go", missing{shouldMiss: true, msg: "Not ready to handle DNE.go"},
		"msg", missing{shouldMiss: true, msg: "Wow, we already have the struct msg"},
		[]string{"Subject", "Body"}, missing{shouldMiss: false, msg: "why don't we have Subject and Body"},
	},
	// missing DNE.go and un-happy about that
	{
		"./DNE.go", missing{shouldMiss: false, msg: "I really need DNE.go"},
		"ReallyImportStruct", missing{shouldMiss: false, msg: "I'm missing the ReallyImportStruct"},
		[]string{"field1", "field2"}, missing{shouldMiss: false, msg: "why don't we have field1 and field2"},
	},

	// have fixture.go and everything else, so super happy
	{
		"./fixture.go", missing{shouldMiss: false, msg: "should have fixture.go"},
		"msg", missing{shouldMiss: false, msg: "why don't we have struct msg?"},
		[]string{"Subject", "Body"}, missing{shouldMiss: false, msg: "why don't we have a Subjeee and Body"},
	},
	// have fixture.go, and un-happy about that only because fields messed up
	{
		"./fixture.go", missing{shouldMiss: false, msg: "should have fixture.go"},
		"msg", missing{shouldMiss: false, msg: "why don't we have struct msg?"},
		[]string{"Subjeee", "Body"}, missing{shouldMiss: false, msg: "why don't we have a Subjeee and Body"},
	},
	// have fixture.go, and un-happy about the fact that struct is missing
	{
		"./fixture.go", missing{shouldMiss: false, msg: "should have fixture.go"},
		"notMSG", missing{shouldMiss: false, msg: "why don't we have struct notMSG?"},
		[]string{"Subject", "Body"}, missing{shouldMiss: false, msg: "why don't we have a Subjeee and Body"},
	},
	// have fixture.go, and un-happy about that and everything else
	{
		"./fixture.go", missing{shouldMiss: true, msg: "Not ready to handle fixture.go"},
		"msg", missing{shouldMiss: true, msg: "Wow, we already have the struct msg"},
		[]string{"Subject", "Body"}, missing{shouldMiss: true, msg: "...and having Subject and Body makes life super difficult"},
	},
	// have fixture.go, and un-happy! But other stuff is just fine
	{
		"./fixture.go", missing{shouldMiss: true, msg: "Not ready to handle fixture.go"},
		"msg", missing{shouldMiss: false, msg: "Might as well have msg"},
		[]string{"Subject", "Body"}, missing{shouldMiss: false, msg: "...and having Subject and Body is fine too"},
	},
}

func TestFixture(t *testing.T) {
	problems := getProblematicExpectations(t, fixtureExpectations)

	require.Equal(t, 7, len(problems))

	require.Equal(t, 0, len(problems[0]))
	require.Equal(t, 1, len(problems[1]))
	require.Equal(t, 0, len(problems[2]))
	require.Equal(t, 1, len(problems[3]))
	require.Equal(t, 1, len(problems[4]))
	require.Equal(t, 3, len(problems[5]))
	require.Equal(t, 1, len(problems[6]))
}

// TestForLint -as the name suggests- is a function meant to get `make lint` to pass
func TestForLint(t *testing.T) {
	m := msg{"hello", nil}
	assert.Equal(t, "hello", m.Subject)
}

Makefile

# includes indexer-v-algod (cf. my json_diff gist)

nightly-setup:
	cd third_party/go-algorand && git pull origin master

nightly-teardown:
	git submodule update

indexer-v-algod-swagger:
	pytest -sv parity

future-nosync:
	go test -run "^(TestTheFuture|TestFixture)$$" github.com/algorand/indexer/future

indexer-v-algod: nightly-setup indexer-v-algod-swagger nightly-teardown

future: nightly-setup future-nosync nightly-teardown

nightly: nightly-setup indexer-v-algod-swagger future-nosync nightly-teardown

.PHONY: indexer-v-algod future

Notable

• ast_expectations.go
  • type expectation struct - together with type missing struct and the handleMissing() method, allows defining what should be expected in a go project and an indifidual script's AST
  • getFileStructs() parses out all the structs inside an AST and outputs a map from struct name to set of struct fields
• ast_expectations_test.go - example usage
• Makefile
  • make future
    1. gets the latest from go-algorand repo in the submodule
    2. runs the tests
    3. restores the go-algorand submodule to its former state