automaton-borkage.log

2021/04/09 11:52:11 INFO fsmsupport.go:252 - *** Entering state SET_KERNELCONFIG after getting event: salt_finished (Src: SALT_COMPUTE_RUNNING)
fatal error: sync: RUnlock of unlocked RWMutex
[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0x11f6601]

goroutine 1 [running]:
runtime.throw(0x14ed190, 0x21)
        /jump/software/rhel7/golang-1.16/src/runtime/panic.go:1117 +0x72 fp=0xc00001b478 sp=0xc00001b448 pc=0x4447f2
sync.throw(0x14ed190, 0x21)
        /jump/software/rhel7/golang-1.16/src/runtime/panic.go:1103 +0x35 fp=0xc00001b498 sp=0xc00001b478 pc=0x4758b5
sync.(*RWMutex).rUnlockSlow(0xc0004fc5d8, 0xffffffff)
        /jump/software/rhel7/golang-1.16/src/sync/rwmutex.go:93 +0x3f fp=0xc00001b4c0 sp=0xc00001b498 pc=0x4922ff
sync.(*RWMutex).RUnlock(0xc0004fc5d8)
        /jump/software/rhel7/golang-1.16/src/sync/rwmutex.go:83 +0x49 fp=0xc00001b4e0 sp=0xc00001b4c0 pc=0x4922a9
runtime.call16(0x0, 0x1569eb8, 0xc00001bb78, 0x800000008)
        /jump/software/rhel7/golang-1.16/src/runtime/asm_amd64.s:550 +0x3e fp=0xc00001b500 sp=0xc00001b4e0 pc=0x47891e
panic(0x1384020, 0x1fc8430)
        /jump/software/rhel7/golang-1.16/src/runtime/panic.go:971 +0x499 fp=0xc00001b5c8 sp=0xc00001b500 pc=0x444319
runtime.panicmem()
        /jump/software/rhel7/golang-1.16/src/runtime/panic.go:212 +0x5b fp=0xc00001b5e8 sp=0xc00001b5c8 pc=0x44243b
runtime.sigpanic()
        /jump/software/rhel7/golang-1.16/src/runtime/signal_unix.go:734 +0x173 fp=0xc00001b620 sp=0xc00001b5e8 pc=0x45aeb3
main.(*NodeMoverFSM).LeaseIsExpired(0xc000784000, 0xc000128847, 0x7, 0xc0000f6630)
        /data/jenkins/jobs/automaton.master/workspace/cmd/nodemoverfsm/main.go:880 +0x201 fp=0xc00001b8b8 sp=0xc00001b620 pc=0x11f6601
main.(*NodeMoverFSM).updateSkynetKernelConfig(0xc000784000)
        /data/jenkins/jobs/automaton.master/workspace/cmd/nodemoverfsm/main.go:634 +0x45 fp=0xc00001b9b8 sp=0xc00001b8b8 pc=0x11f2765
main.(*NodeMoverFSM).updateSkynetKernelConfig-fm()
        /data/jenkins/jobs/automaton.master/workspace/cmd/nodemoverfsm/main.go:632 +0x33 fp=0xc00001b9d0 sp=0xc00001b9b8 pc=0x11f7593
git.w2k.jumptrading.com/linux/automaton/pkg/fsmsupport.M.func1(0xc0004c4070)
        /data/jenkins/jobs/automaton.master/workspace/pkg/fsmsupport/fsmsupport.go:171 +0x1a5 fp=0xc00001ba50 sp=0xc00001b9d0 pc=0x11ea025
github.com/looplab/fsm.(*FSM).enterStateCallbacks(0xc0004fc5a0, 0xc0004c4070)
        /data/jenkins/go/pkg/mod/github.com/looplab/[email protected]/fsm.go:403 +0x102 fp=0xc00001baa0 sp=0xc00001ba50 pc=0x11e0702
github.com/looplab/fsm.(*FSM).Event.func1()
        /data/jenkins/go/pkg/mod/github.com/looplab/[email protected]/fsm.go:308 +0x9e fp=0xc00001bae8 sp=0xc00001baa0 pc=0x11e11be
github.com/looplab/fsm.transitionerStruct.transition(...)
        /data/jenkins/go/pkg/mod/github.com/looplab/[email protected]/fsm.go:354
github.com/looplab/fsm.(*transitionerStruct).transition(0x2071af8, 0xc0004fc5a0, 0x0, 0x0)
        <autogenerated>:1 +0x64 fp=0xc00001baf8 sp=0xc00001bae8 pc=0x11e1264
github.com/looplab/fsm.(*FSM).doTransition(...)
        /data/jenkins/go/pkg/mod/github.com/looplab/[email protected]/fsm.go:339
github.com/looplab/fsm.(*FSM).Event(0xc0004fc5a0, 0x14d717a, 0xd, 0x0, 0x0, 0x0, 0x0, 0x0)
        /data/jenkins/go/pkg/mod/github.com/looplab/[email protected]/fsm.go:321 +0x5c2 fp=0xc00001bcb8 sp=0xc00001baf8 pc=0x11dfc22
git.w2k.jumptrading.com/linux/automaton/pkg/fsmsupport.(*CommonFSM).Launch(0xc000784078, 0x14d016b, 0x4)
        /data/jenkins/jobs/automaton.master/workspace/pkg/fsmsupport/fsmsupport.go:291 +0x19f fp=0xc00001bd48 sp=0xc00001bcb8 pc=0x11e301f
main.main()
        /data/jenkins/jobs/automaton.master/workspace/cmd/nodemoverfsm/main.go:945 +0x553 fp=0xc00001bf88 sp=0xc00001bd48 pc=0x11f6c53
runtime.main()
        /jump/software/rhel7/golang-1.16/src/runtime/proc.go:225 +0x256 fp=0xc00001bfe0 sp=0xc00001bf88 pc=0x447036
runtime.goexit()
        /jump/software/rhel7/golang-1.16/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc00001bfe8 sp=0xc00001bfe0 pc=0x47a381
goroutine 6 [chan receive]:
k8s.io/klog.(*loggingT).flushDaemon(0x2041de0)
        /data/jenkins/go/pkg/mod/k8s.io/[email protected]/klog.go:1010 +0x8b
created by k8s.io/klog.init.0
        /data/jenkins/go/pkg/mod/k8s.io/[email protected]/klog.go:411 +0xd8

goroutine 37 [select, 5 minutes]:
database/sql.(*DB).connectionOpener(0xc0004cbe10, 0x1676bd8, 0xc0007a6040)
        /jump/software/rhel7/golang-1.16/src/database/sql/sql.go:1133 +0xb5
created by database/sql.OpenDB
        /jump/software/rhel7/golang-1.16/src/database/sql/sql.go:740 +0x12a

goroutine 59 [select, 5 minutes]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1(0xc00010e300, 0xc0004cf2c0, 0xc0001302a0)
        /data/jenkins/go/pkg/mod/github.com/go-sql-driver/[email protected]/connection_go18.go:178 +0xa5
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
        /data/jenkins/go/pkg/mod/github.com/go-sql-driver/[email protected]/connection_go18.go:175 +0xbe

goroutine 70 [select, 5 minutes]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1(0xc00010e8a0, 0xc00014e000, 0xc000130780)
        /data/jenkins/go/pkg/mod/github.com/go-sql-driver/[email protected]/connection_go18.go:178 +0xa5
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
        /data/jenkins/go/pkg/mod/github.com/go-sql-driver/[email protected]/connection_go18.go:175 +0xbe

goroutine 60 [select, 5 minutes]:
database/sql.(*DB).connectionOpener(0xc000138680, 0x1676bd8, 0xc0001341c0)
        /jump/software/rhel7/golang-1.16/src/database/sql/sql.go:1133 +0xb5
created by database/sql.OpenDB
        /jump/software/rhel7/golang-1.16/src/database/sql/sql.go:740 +0x12a

goroutine 243 [IO wait]:
internal/poll.runtime_pollWait(0x7fffce054e28, 0x72, 0xffffffffffffffff)
        /jump/software/rhel7/golang-1.16/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc00012e298, 0x72, 0xb00, 0xbb9, 0xffffffffffffffff)
        /jump/software/rhel7/golang-1.16/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
        /jump/software/rhel7/golang-1.16/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00012e280, 0xc0002a2c00, 0xbb9, 0xbb9, 0x0, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc00012e280, 0xc0002a2c00, 0xbb9, 0xbb9, 0xbb4, 0xc0002a2c00, 0x5)
        /jump/software/rhel7/golang-1.16/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000114298, 0xc0002a2c00, 0xbb9, 0xbb9, 0x0, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/net/net.go:183 +0x91
crypto/tls.(*atLeastReader).Read(0xc0006ee138, 0xc0002a2c00, 0xbb9, 0xbb9, 0xbb4, 0x2042660, 0x0)
        /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:776 +0x63
bytes.(*Buffer).ReadFrom(0xc000784cf8, 0x164ea60, 0xc0006ee138, 0x419c85, 0x13962e0, 0x147ed80)
        /jump/software/rhel7/golang-1.16/src/bytes/buffer.go:204 +0xbe
crypto/tls.(*Conn).readFromUntil(0xc000784a80, 0x1650080, 0xc000114298, 0x5, 0xc000114298, 0x1b8)
        /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:798 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc000784a80, 0x0, 0x0, 0x8)
        /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:605 +0x115
crypto/tls.(*Conn).readRecord(...)
        /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:573
crypto/tls.(*Conn).Read(0xc000784a80, 0xc000261000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:1276 +0x165
bufio.(*Reader).Read(0xc0004fd740, 0xc00025e3b8, 0x9, 0x9, 0x77ceeb, 0xc0002c0c78, 0x415505)
        /jump/software/rhel7/golang-1.16/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x164e920, 0xc0004fd740, 0xc00025e3b8, 0x9, 0x9, 0x9, 0xc000264880, 0xe6145989b0c100, 0xc000264880)
        /jump/software/rhel7/golang-1.16/src/io/io.go:328 +0x87
io.ReadFull(...)
        /jump/software/rhel7/golang-1.16/src/io/io.go:347
net/http.http2readFrameHeader(0xc00025e3b8, 0x9, 0x9, 0x164e920, 0xc0004fd740, 0x0, 0x0, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:1477 +0x89
net/http.(*http2Framer).ReadFrame(0xc00025e380, 0xc0004aec90, 0x0, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:1735 +0xa5
net/http.(*http2clientConnReadLoop).run(0xc0002c0fa8, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:8322 +0xd8
net/http.(*http2ClientConn).readLoop(0xc000582900)
        /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:8244 +0x6f
created by net/http.(*http2Transport).newClientConn
        /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:7208 +0x6c5

goroutine 179 [IO wait]:
internal/poll.runtime_pollWait(0x7fffce054ff8, 0x72, 0xffffffffffffffff)
        /jump/software/rhel7/golang-1.16/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc000244698, 0x72, 0xb00, 0xbbf, 0xffffffffffffffff)
        /jump/software/rhel7/golang-1.16/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
        /jump/software/rhel7/golang-1.16/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000244680, 0xc0002a3800, 0xbbf, 0xbbf, 0x0, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc000244680, 0xc0002a3800, 0xbbf, 0xbbf, 0xbba, 0xc0002a3800, 0x5)
        /jump/software/rhel7/golang-1.16/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000010368, 0xc0002a3800, 0xbbf, 0xbbf, 0x0, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/net/net.go:183 +0x91
crypto/tls.(*atLeastReader).Read(0xc000722ea0, 0xc0002a3800, 0xbbf, 0xbbf, 0xc000276000, 0xc000700000, 0x0)
        /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:776 +0x63
bytes.(*Buffer).ReadFrom(0xc000784978, 0x164ea60, 0xc000722ea0, 0x419c85, 0x13962e0, 0x147ed80)
        /jump/software/rhel7/golang-1.16/src/bytes/buffer.go:204 +0xbe
crypto/tls.(*Conn).readFromUntil(0xc000784700, 0x1650080, 0xc000010368, 0x5, 0xc000010368, 0x400)
        /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:798 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc000784700, 0x0, 0x0, 0x1e)
        /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:605 +0x115
crypto/tls.(*Conn).readRecord(...)
        /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:573
crypto/tls.(*Conn).Read(0xc000784700, 0xc000149000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:1276 +0x165
bufio.(*Reader).Read(0xc00010fa40, 0xc00025e2d8, 0x9, 0x9, 0x77ceeb, 0xc0002c1c78, 0x415505)
        /jump/software/rhel7/golang-1.16/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x164e920, 0xc00010fa40, 0xc00025e2d8, 0x9, 0x9, 0x9, 0xc0005d3c80, 0xe6145989b0c100, 0xc0005d3c80)
        /jump/software/rhel7/golang-1.16/src/io/io.go:328 +0x87
io.ReadFull(...)
        /jump/software/rhel7/golang-1.16/src/io/io.go:347
net/http.http2readFrameHeader(0xc00025e2d8, 0x9, 0x9, 0x164e920, 0xc00010fa40, 0x0, 0x0, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:1477 +0x89
net/http.(*http2Framer).ReadFrame(0xc00025e2a0, 0xc0001bf350, 0x0, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:1735 +0xa5
net/http.(*http2clientConnReadLoop).run(0xc0002c1fa8, 0x0, 0x0)
        /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:8322 +0xd8
net/http.(*http2ClientConn).readLoop(0xc000683680)
        /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:8244 +0x6f
created by net/http.(*http2Transport).newClientConn
        /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:7208 +0x6c5