mdatagen_unmarshal_logs.md

Without unmarshal I get this

sub, err := cm.Sub(tt.id.String())
require.NoError(t, err)
require.NoError(t, component.UnmarshalConfig(sub, cfg))

assert.NoError(t, component.ValidateConfig(cfg))
if diff := cmp.Diff(tt.expected, cfg, cmpopts.IgnoreUnexported(metadata.MetricsBuilderConfig{}), cmpopts.IgnoreUnexported(metadata.MetricSettings{})); diff != "" {
   t.Errorf("Config mismatch (-expected +actual):\n%s", diff)
}

Which uses he default unmarshaler

// UnmarshalConfig helper function to UnmarshalConfig a Config.
// It checks if the config implements confmap.Unmarshaler and uses that if available,
// otherwise uses Map.UnmarshalExact, erroring if a field is nonexistent.
func UnmarshalConfig(conf *confmap.Conf, intoCfg Config) error {
   if cu, ok := intoCfg.(confmap.Unmarshaler); ok {
      return cu.Unmarshal(conf)
   }

   return conf.Unmarshal(intoCfg, confmap.WithErrorUnused())
}

this all gives the following error

--- FAIL: TestLoadConfig (0.00s)
    --- FAIL: TestLoadConfig/active_directory_ds (0.00s)
        config_test.go:67:
                Error Trace:    /home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/activedirectorydsreceiver/config_test.go:67
                Error:          Received unexpected error:
                                1 error(s) decoding:

                                * '' has invalid keys: collection_interval
                Test:           TestLoadConfig/active_directory_ds

Tests pass with this:

func (cfg *Config) Unmarshal(parser *confmap.Conf) error {
   if parser == nil {
      return nil
   }
   err := parser.Unmarshal(cfg) // , confmap.WithErrorUnused()) // , cmpopts.IgnoreUnexported(metadata.MetricSettings{}))
   if err != nil {
      return err
   }
   return nil
}

giving the output

❯ make gotest
make for-all-target TARGET="test"
make[1]: Entering directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib'
Running target 'test' in module 'receiver/activedirectorydsreceiver' as part of group 'all'
make -C receiver/activedirectorydsreceiver test
make[2]: Entering directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/activedirectorydsreceiver'
if [ "" = "true" ]; then \
        go test -race -timeout 300s -parallel 4 --tags="" -v ./... 2>&1 | tee -a ./foresight-test-report.txt; \
else \
        go test -race -timeout 300s -parallel 4 --tags="" ./...; \
fi
ok      github.com/open-telemetry/opentelemetry-collector-contrib/receiver/activedirectorydsreceiver    0.049s
ok      github.com/open-telemetry/opentelemetry-collector-contrib/receiver/activedirectorydsreceiver/internal/metadata  (cached)
make[2]: Leaving directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/activedirectorydsreceiver'
Running target 'test' in module 'receiver/aerospikereceiver' as part of group 'all'
make -C receiver/aerospikereceiver test
make[2]: Entering directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/aerospikereceiver'
if [ "" = "true" ]; then \
        go test -race -timeout 300s -parallel 4 --tags="" -v ./... 2>&1 | tee -a ./foresight-test-

and it'll continue to run until

Running target 'test' in module 'receiver/bigipreceiver' as part of group 'all'
make -C receiver/bigipreceiver test
make[2]: Entering directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/bigipreceiver'
if [ "" = "true" ]; then \
        go test -race -timeout 300s -parallel 4 --tags="" -v ./... 2>&1 | tee -a ./foresight-test-report.txt; \
else \
        go test -race -timeout 300s -parallel 4 --tags="" ./...; \
fi
--- FAIL: TestLoadConfig (0.00s)
    config_test.go:140:
                Error Trace:    /home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/bigipreceiver/config_test.go:140
                Error:          Received unexpected error:
                                1 error(s) decoding:

                                * '' has invalid keys: collection_interval, endpoint, password, tls, username
                Test:           TestLoadConfig

which shows it's an issue in unmarshaling. Adding confmap to the bigip receiver's gomod will do this:

Running target 'test' in module 'receiver/bigipreceiver' as part of group 'all'
make -C receiver/bigipreceiver test
make[2]: Entering directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/bigipreceiver'
if [ "" = "true" ]; then \
        go test -race -timeout 300s -parallel 4 --tags="" -v ./... 2>&1 | tee -a ./foresight-test-report.txt; \
else \
        go test -race -timeout 300s -parallel 4 --tags="" ./...; \
fi
ok      github.com/open-telemetry/opentelemetry-collector-contrib/receiver/bigipreceiver        0.199s
ok      github.com/open-telemetry/opentelemetry-collector-contrib/receiver/bigipreceiver/internal/metadata      0.066s
?       github.com/open-telemetry/opentelemetry-collector-contrib/receiver/bigipreceiver/internal/mocks [no test files]
?       github.com/open-telemetry/opentelemetry-collector-contrib/receiver/bigipreceiver/internal/models        [no test files]
make[2]: Leaving directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/bigipreceiver'
Running target 'test' in module 'receiver/carbonreceiver' as part of group 'all'
make -C receiver/carbonreceiver test
make[2]: Entering directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/carbonreceiver'
if [ "" = "true" ]; then \
        go test -race -timeout 300s -parallel 4 --tags="" -v ./... 2>&1 | tee -a ./foresight-test-report.txt; \
else \
        go test -race -timeout 300s -parallel 4 --tags="" ./...; \
fi
ok      github.com/open-telemetry/opentelemetry-collector-contrib/receiver/carbonreceiver       0.057s
ok      github.com/open-telemetry/opentelemetry-collec

until it fails with another config parse issue at

make[2]: Leaving directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/carbonreceiver'
Running target 'test' in module 'receiver/chronyreceiver' as part of group 'all'
make -C receiver/chronyreceiver test
make[2]: Entering directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/chronyreceiver'
if [ "" = "true" ]; then \
        go test -race -timeout 300s -parallel 4 --tags="" -v ./... 2>&1 | tee -a ./foresight-test-report.txt; \
else \
        go test -race -timeout 300s -parallel 4 --tags="" ./...; \
fi
--- FAIL: TestLoadConfig (0.00s)
    config_test.go:45:
                Error Trace:    /home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/chronyreceiver/config_test.go:45
                Error:          Received unexpected error:
                                1 error(s) decoding:

                                * '' has invalid keys: endpoint, timeout
                Test:           TestLoadConfig
FAIL
FAIL    github.com/open-telemetry/opentelemetry-collector-contrib/receiver/chronyreceiver       0.023s
ok      github.com/open-telemetry/opentelemetry-collector-contrib/receiver/chronyreceiver/internal/chrony       0.139s
ok      github.com/open-telemetry/opentelemetry-collector-contrib/receiver/chronyreceiver/internal/metadata     0.045s
FAIL
make[2]: *** [../../Makefile.Common:98: test] Error 1
make[2]: Leaving directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/chronyreceiver'
make[1]: *** [Makefile:167: receiver/chronyreceiver] Error 2
make[1]: Leaving directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib'
make: *** [Makefile:97: gotest] Error 2
    ~/works/u/o/opentelemetry-collector-contrib    MetricsSetti…shal_no_with !6 ?2 

This is the script I ran on the last PR

❯ cat add_unmarshal.sh
#!/bin/zsh
rm -Rf .tmp-renames; mkdir .tmp-renames
find receiver -name config.go \
    -exec grep -L Unmarshal {} \; \
    -exec sh -c 'cat unmarshal_normal.txt >> {} && gofmt -w -s {}' \; \
    -exec grep -L "go.opentelemetry.io/collector/confmap" {} \; \
    -exec sh -c 'cd $(dirname {}) && go get -x "go.opentelemetry.io/collector/confmap" && cd -' \; \
    -exec sh -c "sed -i '/^package.*/a import (\"go.opentelemetry.io/collector/confmap\")' {} " \; \
    -exec gofmt -w -s {} \; \
    -exec goimports -w -local "github.com/open-telemetry/opentelemetry-collector-contrib" {} \;  \
    -exec ./.tools/gci write {} --skip-generated -s standard,default,prefix\(github.com/open-telemetry/opentelemetry-collector-contrib\) \;


# this command
grep -E -h -o "func \(.*(\*)?Config\)" ./**/config.go  | cat | sort | uniq -c
# returns
echo "still need to fix these"
find receiver -name config.go -exec grep -l -E 'func \(rCfg (\*)?Config\)' {} \; -exec sh -c "sed -i 's/cfg/rCfg/g' {}" \; -print
find receiver -name config.go -exec grep -l -E 'func \(c (\*)?Config\)' {} \; -exec sh -c "sed -i 's/cfg/c/g' {}" \; -print
find receiver -name config.go -exec grep -l -E 'func \(config (\*)?(Group)?Config\)' {} \; -exec sh -c "sed -i 's/cfg/config/g' {}" \; -print
#
# From here manually edit docker and prometheus to have cfg and config or whatever in their config.go files

# current state is jmx failing unit test lookup at a nil blah blah not sure how real it is


# -exec ./.tools/gci --skip-generated -s standard,default,prefix\(github.com/open-telemetry/opentelemetry-collector-contrib\) {} \;
rm -Rf .tmp-renames;

I could change the generated Unmarshal to this,

func (mbc *MetricsBuilderConfig) Unmarshal(parser *confmap.Conf) error {
	if parser == nil {
		return nil
	}
	err := parser.Unmarshal(mbc)
	if err != nil {
		return err
	}
	return nil
}

but this still causes issues

                                Diff:
                                --- Expected
                                +++ Actual
                                @@ -38,3 +38,3 @@
                                  },
                                - Endpoint: (string) (len=20) "udp://localhost:3030",
                                + Endpoint: (string) (len=35) "unix:///var/run/chrony/chronyd.sock",
                                  Timeout: (time.Duration) 10000000000
                Test:           TestLoadConfig

but it's weird because the original issue is

--- FAIL: TestLoadConfig (0.00s)
    config_test.go:45: 
                Error Trace:    /home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/chronyreceiver/config_test.go:45
                Error:          Received unexpected error:
                                1 error(s) decoding:
                                
                                * '' has invalid keys: endpoint, timeout
                Test:           TestLoadConfig

yet timeout seems correctly set?

Adding in the Unmarshal from my script to Config.go will let the test pass, however.

func (c *Config) Unmarshal(parser *confmap.Conf) error {
	if parser == nil {
		return nil
	}
	err := parser.Unmarshal(c, confmap.WithErrorUnused())
	if err != nil {
		return err
	}
	return nil
}

ahh, can't edit gists.

@ initial post:

sub, err := cm.Sub(tt.id.String())
require.NoError(t, err)
require.NoError(t, component.UnmarshalConfig(sub, cfg))

assert.NoError(t, component.ValidateConfig(cfg))
if diff := cmp.Diff(tt.expected, cfg, cmpopts.IgnoreUnexported(metadata.MetricsBuilderConfig{}), cmpopts.IgnoreUnexported(metadata.MetricSettings{})); diff != "" {
   t.Errorf("Config mismatch (-expected +actual):\n%s", diff)
}

it specifically errors out on this line

require.NoError(t, component.UnmarshalConfig(sub, cfg))

Alright, reset my state. I got things building when removing the custom Unmarshals for MetricsBuilderConfig and ResourceAttributesSettings in mdatagen's metrics.go.tmpl. This is fine for now, but if we ever want to add enabledSetbyUser for type ResourceAttributeSettings, we'll need to add an Unmarshal back for ResourceAttributeSettings lest we get the expected error

ok      github.com/open-telemetry/opentelemetry-collector-contrib/receiver/datadogreceiver      (cached)
make[2]: Leaving directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/datadogreceiver'
Running target 'test' in module 'receiver/dockerstatsreceiver' as part of group 'all'
make -C receiver/dockerstatsreceiver test
make[2]: Entering directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/dockerstatsreceiver'
if [ "" = "true" ]; then \
        go test -race -timeout 300s -parallel 4 --tags="" -v ./... 2>&1 | tee -a ./foresight-test-report.txt; \
else \
        go test -race -timeout 300s -parallel 4 --tags="" ./...; \
fi
--- FAIL: TestLoadConfig (0.00s)
    --- FAIL: TestLoadConfig/docker_stats (0.00s)
panic: cannot handle unexported field at {*dockerstatsreceiver.Config}.MetricsBuilderConfig.ResourceAttributes.ContainerHostname.enabledSetByUser:
        "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver/internal/metadata".ResourceAttributeSettings
consider using a custom Comparer; if you control the implementation of type, you can also consider using an Exporter, AllowUnexported, or cmpopts.IgnoreUnexported [recovered]
        panic: cannot handle unexported field at {*dockerstatsreceiver.Config}.MetricsBuilderConfig.ResourceAttributes.ContainerHostname.enabledSetByUser:
        "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver/internal/metadata".ResourceAttributeSettings
consider using a custom Comparer; if you control the implementation of type, you can also consider using an Exporter, AllowUnexported, or cmpopts.IgnoreUnexported

goroutine 132 [running]:
testing.tRunner.func1.2({0x1192c20, 0xc000121760})
        /home/jameshughes/.asdf/installs/golang/1.19.2/go/src/testing/testing.go:1396 +0x372
testing.tRunner.func1()
        /home/jameshughes/.asdf/installs/golang/1.19.2/go/src/testing/testing.go:1399 +0x5f0
panic({0x1192c20, 0xc000121760})
        /home/jameshughes/.asdf/installs/golang/1.19.2/go/src/runtime/panic.go:890 +0x262
github.com/google/go-cmp/cmp.validator.apply({{}}, 0xc000466d20, {0x11843e0?, 0xc0004dc0cf?, 0x1493998?}, {0x11843e0?, 0xc0004dc28f?, 0x11843e0?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/options.go:246 +0x445
github.com/google/go-cmp/cmp.(*state).tryOptions(0xc000466d20, {0x14a3c20, 0x11843e0}, {0x11843e0?, 0xc0004dc0cf?, 0x1072fe5?}, {0x11843e0?, 0xc0004dc28f?, 0xc0004e45b0?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:304 +0x146
github.com/google/go-cmp/cmp.(*state).compareAny(0xc000466d20, {0x1496f78, 0xc0000f6a00?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:259 +0x5a9
github.com/google/go-cmp/cmp.(*state).compareStruct(0xc000466d20, {0x14a3c20, 0x12201e0}, {0x12201e0?, 0xc0004dc0ce?, 0x1072fe5?}, {0x12201e0?, 0xc0004dc28e?, 0xc0004e4a10?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:412 +0xa52
github.com/google/go-cmp/cmp.(*state).compareAny(0xc000466d20, {0x1496f78, 0xc0000f6900?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:287 +0x1412
github.com/google/go-cmp/cmp.(*state).compareStruct(0xc000466d20, {0x14a3c20, 0x127b240}, {0x127b240?, 0xc0004dc0ce?, 0x1072fe5?}, {0x127b240?, 0xc0004dc28e?, 0xc0004e4e70?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:412 +0xa52
github.com/google/go-cmp/cmp.(*state).compareAny(0xc000466d20, {0x1496f78, 0xc0006c0800?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:287 +0x1412
github.com/google/go-cmp/cmp.(*state).compareStruct(0xc000466d20, {0x14a3c20, 0x1220140}, {0x1220140?, 0xc0004dc050?, 0x4a8bc5?}, {0x1220140?, 0xc0004dc210?, 0xc0004e52d0?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:412 +0xa52
github.com/google/go-cmp/cmp.(*state).compareAny(0xc000466d20, {0x1496f78, 0xc0006c0600?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:287 +0x1412
github.com/google/go-cmp/cmp.(*state).compareStruct(0xc000466d20, {0x14a3c20, 0x129b820}, {0x129b820?, 0xc0004dc000?, 0x203000?}, {0x129b820?, 0xc0004dc1c0?, 0x107304b?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:412 +0xa52
github.com/google/go-cmp/cmp.(*state).compareAny(0xc000466d20, {0x1496ee8, 0xc0004c49c0?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:287 +0x1412
github.com/google/go-cmp/cmp.(*state).comparePtr(0xc000466d20, {0x14a3c20, 0x11bec00}, {0x11bec00?, 0xc0004dc000?, 0x0?}, {0x11bec00?, 0xc0004dc1c0?, 0xc0004c4980?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:564 +0x785
github.com/google/go-cmp/cmp.(*state).compareAny(0xc000466d20, {0x1495178, 0xc0004c4980?})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:293 +0x1252
github.com/google/go-cmp/cmp.Diff({0x11bec00, 0xc0004dc000}, {0x11bec00, 0xc0004dc1c0}, {0xc000121690, 0x1, 0x1})
        /home/jameshughes/.go/.otel/pkg/mod/github.com/google/[email protected]/cmp/compare.go:120 +0xd5
github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver.TestLoadConfig.func2(0xc000681380)
        /home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/dockerstatsreceiver/config_test.go:96 +0x2a6
testing.tRunner(0xc000681380, 0xc000120390)
        /home/jameshughes/.asdf/installs/golang/1.19.2/go/src/testing/testing.go:1446 +0x217
created by testing.(*T).Run
        /home/jameshughes/.asdf/installs/golang/1.19.2/go/src/testing/testing.go:1493 +0x75e
FAIL    github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver  0.051s
ok      github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver/internal/metadata        0.076s
FAIL
make[2]: *** [../../Makefile.Common:98: test] Error 1
make[2]: Leaving directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib/receiver/dockerstatsreceiver'
make[1]: *** [Makefile:167: receiver/dockerstatsreceiver] Error 2
make[1]: Leaving directory '/home/jameshughes/workspace/unison/otel-github/opentelemetry-collector-contrib'
make: *** [Makefile:97: gotest] Error 2