gistfile1.txt

2014/06/24 03:15:47 rpc: client protocol error: unexpected EOF
panic: runtime error: invalid memory address or nil pointer dereference
[signal 0xb code=0x1 addr=0x40 pc=0x4070a1]

goroutine 130 [running]:
runtime.panic(0x6c2440, 0xa63b28)
	/usr/local/go/src/pkg/runtime/panic.c:266 +0xb6
main.(*Formation).remove(0xc2100f4880, 0xa, 0xc2100f5a30, 0x3)
	/vagrant/src/flynn-controller/scheduler/main.go:619 +0x3e1
main.(*Formation).rectify(0xc2100f4880)
	/vagrant/src/flynn-controller/scheduler/main.go:521 +0x72b
main.(*Formation).Rectify(0xc2100f4880)
	/vagrant/src/flynn-controller/scheduler/main.go:463 +0x5f
main.func·001()
	/vagrant/src/flynn-controller/scheduler/main.go:190 +0x37
created by main.(*context).watchFormations
	/vagrant/src/flynn-controller/scheduler/main.go:194 +0x43d

goroutine 1 [chan receive]:
main.(*context).watchFormations(0xc210022e10, 0x0)
	/vagrant/src/flynn-controller/scheduler/main.go:175 +0x1a0
main.main()
	/vagrant/src/flynn-controller/scheduler/main.go:47 +0x5d9

goroutine 4 [IO wait]:
net.runtime_pollWait(0x7f7f880346a8, 0x72, 0x0)
	/usr/local/go/src/pkg/runtime/netpoll.goc:116 +0x6a
net.(*pollDesc).Wait(0xc21004dca0, 0x72, 0x7f7f88033170, 0xb)
	/usr/local/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
net.(*pollDesc).WaitRead(0xc21004dca0, 0xb, 0x7f7f88033170)
	/usr/local/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
net.(*netFD).Read(0xc21004dc40, 0xc21006c000, 0x1000, 0x1000, 0x0, ...)
	/usr/local/go/src/pkg/net/fd_unix.go:204 +0x2a0
net.(*conn).Read(0xc210000370, 0xc21006c000, 0x1000, 0x1000, 0x7f7f8802a000, ...)
	/usr/local/go/src/pkg/net/net.go:122 +0xc5
now=2014-06-24T03:15:47+0000 app=controller-scheduler fn=remove app.id=1c709bf4f3d046b2b0ee080d230921ae release.id=2f013cc260c24e1cabfc3ea779b42d8f host.id=acd6ea35cf65 job.id=d10a6b4a257341028cf465c592ff0d5a
bufio.(*Reader).fill(0xc210041480)
	/usr/local/go/src/pkg/bufio/bufio.go:91 +0x110
bufio.(*Reader).Read(0xc210041480, 0xc21000acb0, 0x1, 0x9, 0x1, ...)
	/usr/local/go/src/pkg/bufio/bufio.go:159 +0x1a4
io.ReadAtLeast(0x7f7f88034888, 0xc210041480, 0xc21000acb0, 0x1, 0x9, ...)
	/usr/local/go/src/pkg/io/io.go:288 +0xf6
io.ReadFull(0x7f7f88034888, 0xc210041480, 0xc21000acb0, 0x1, 0x9, ...)
	/usr/local/go/src/pkg/io/io.go:306 +0x71
encoding/gob.decodeUintReader(0x7f7f88034888, 0xc210041480, 0xc21000acb0, 0x9, 0x9, ...)
	/usr/local/go/src/pkg/encoding/gob/decode.go:66 +0x98
encoding/gob.(*Decoder).recvMessage(0xc21000b0f0, 0xc21006f750)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:73 +0x57
encoding/gob.(*Decoder).decodeTypeSequence(0xc21000b0f0, 0xc21006e200, 0x160)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:159 +0x49
encoding/gob.(*Decoder).DecodeValue(0xc21000b0f0, 0x6238a0, 0xc21006e270, 0x160, 0x0, ...)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:223 +0x12e
encoding/gob.(*Decoder).Decode(0xc21000b0f0, 0x6238a0, 0xc21006e270, 0xc21004f218, 0xc2101444d0)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:202 +0x1c5
github.com/flynn/rpcplus.(*gobClientCodec).ReadResponseHeader(0xc210065db0, 0xc21006e270, 0xc210073520, 0x0)
	/tmp/godep/rev/05/df3d30b6e55f5b440c1a78e2f83cb822bc0d76/src/github.com/flynn/rpcplus/client.go:278 +0x3f
github.com/flynn/rpcplus.(*Client).input(0xc2100414e0)
	/tmp/godep/rev/05/df3d30b6e55f5b440c1a78e2f83cb822bc0d76/src/github.com/flynn/rpcplus/client.go:138 +0xb8
created by github.com/flynn/rpcplus.NewClientWithCodec
	/tmp/godep/rev/05/df3d30b6e55f5b440c1a78e2f83cb822bc0d76/src/github.com/flynn/rpcplus/client.go:256 +0x91

goroutine 5 [chan receive]:
github.com/flynn/go-discoverd.func·001()
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:123 +0x5b
created by github.com/flynn/go-discoverd.(*serviceSet).bind
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:165 +0xcf

goroutine 6 [chan receive]:
github.com/flynn/go-flynn/cluster.(*Client).followLeader(0xc21004f3f0, 0x0)
	/tmp/godep/rev/77/5d010f4f7ff91b144418ad74c6f6e285d50f49/src/github.com/flynn/go-flynn/cluster/client.go:90 +0x50
created by github.com/flynn/go-flynn/cluster.(*Client).start
	/tmp/godep/rev/77/5d010f4f7ff91b144418ad74c6f6e285d50f49/src/github.com/flynn/go-flynn/cluster/client.go:85 +0x53

goroutine 7 [chan receive]:
github.com/flynn/go-discoverd.func·002()
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:224 +0x8d
created by github.com/flynn/go-discoverd.(*serviceSet).Leaders
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:230 +0xea

goroutine 8 [runnable]:
net.runtime_pollWait(0x7f7f88034600, 0x72, 0x0)
	/usr/local/go/src/pkg/runtime/netpoll.goc:116 +0x6a
net.(*pollDesc).Wait(0xc210072450, 0x72, 0x7f7f88033170, 0xb)
	/usr/local/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
net.(*pollDesc).WaitRead(0xc210072450, 0xb, 0x7f7f88033170)
	/usr/local/go/src/pkg/net/fd_poll_runtime.go:86 +0x30
net.(*netFD).Read(0xc2100723f0, 0xc21007a000, 0x1000, 0x1000, 0x0, ...)
	/usr/local/go/src/pkg/net/fd_unix.go:204 +0x2a0
net.(*conn).Read(0xc210000b58, 0xc21007a000, 0x1000, 0x1000, 0xc210187600, ...)
	/usr/local/go/src/pkg/net/net.go:122 +0xc5
bufio.(*Reader).fill(0xc210041d80)
	/usr/local/go/src/pkg/bufio/bufio.go:91 +0x110
bufio.(*Reader).Read(0xc210041d80, 0xc210073450, 0x1, 0x9, 0x1, ...)
	/usr/local/go/src/pkg/bufio/bufio.go:159 +0x1a4
io.ReadAtLeast(0x7f7f88034888, 0xc210041d80, 0xc210073450, 0x1, 0x9, ...)
	/usr/local/go/src/pkg/io/io.go:288 +0xf6
io.ReadFull(0x7f7f88034888, 0xc210041d80, 0xc210073450, 0x1, 0x9, ...)
	/usr/local/go/src/pkg/io/io.go:306 +0x71
encoding/gob.decodeUintReader(0x7f7f88034888, 0xc210041d80, 0xc210073450, 0x9, 0x9, ...)
	/usr/local/go/src/pkg/encoding/gob/decode.go:66 +0x98
encoding/gob.(*Decoder).recvMessage(0xc21000b2d0, 0xc210124000)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:73 +0x57
encoding/gob.(*Decoder).decodeTypeSequence(0xc21000b2d0, 0xc210077400, 0x160)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:159 +0x49
encoding/gob.(*Decoder).DecodeValue(0xc21000b2d0, 0x6238a0, 0xc2100774e0, 0x160, 0x0, ...)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:223 +0x12e
encoding/gob.(*Decoder).Decode(0xc21000b2d0, 0x6238a0, 0xc2100774e0, 0xc21004f680, 0xc210144700)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:202 +0x1c5
github.com/flynn/rpcplus.(*gobClientCodec).ReadResponseHeader(0xc210077480, 0xc2100774e0, 0xc2100f5ec0, 0x0)
	/tmp/godep/rev/05/df3d30b6e55f5b440c1a78e2f83cb822bc0d76/src/github.com/flynn/rpcplus/client.go:278 +0x3f
github.com/flynn/rpcplus.(*Client).input(0xc210041de0)
	/tmp/godep/rev/05/df3d30b6e55f5b440c1a78e2f83cb822bc0d76/src/github.com/flynn/rpcplus/client.go:138 +0xb8
created by github.com/flynn/rpcplus.NewClientWithCodec
	/tmp/godep/rev/05/df3d30b6e55f5b440c1a78e2f83cb822bc0d76/src/github.com/flynn/rpcplus/client.go:256 +0x91

goroutine 9 [select]:
github.com/flynn/go-discoverd.func·005()
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:491 +0x2e8
created by github.com/flynn/go-discoverd.(*Client).RegisterWithAttributes
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:502 +0x454

goroutine 10 [chan receive]:
github.com/flynn/go-discoverd.func·001()
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:123 +0x5b
created by github.com/flynn/go-discoverd.(*serviceSet).bind
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:165 +0xcf

goroutine 14 [chan receive]:
github.com/flynn/go-discoverd.func·002()
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:224 +0x8d
created by github.com/flynn/go-discoverd.(*serviceSet).Leaders
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:230 +0xea

goroutine 15 [chan receive]:
github.com/flynn/go-discoverd.func·001()
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:123 +0x5b
created by github.com/flynn/go-discoverd.(*serviceSet).bind
	/tmp/godep/rev/ef/8e9c70ed6335e8db5a66addfa96226f35e2bd3/src/github.com/flynn/go-discoverd/client.go:165 +0xcf

goroutine 16 [IO wait]:
net.runtime_pollWait(0x7f7f88034558, 0x72, 0x0)
	/usr/local/go/src/pkg/runtime/netpoll.goc:116 +0x6a
net.(*pollDesc).Wait(0xc210072840, 0x72, 0x7f7f88033170, 0xb)
	/usr/local/go/src/pkg/net/fd_poll_runtime.go:81 +0x34
net.(*pollDesc).WaitRead(0xc210072840, 0xb, 0x7f7f88033170)
	/usr/local/go/src/pkg/net/fd_poll_runtime.go:86 +0x30

func main() {
        grohl.AddContext("app", "controller-scheduler")
        grohl.Log(grohl.Data{"at": "start"})

        cc, err := controller.NewClient("", os.Getenv("AUTH_KEY"))
        if err != nil {
                log.Fatal(err)
        }
        cl, err := cluster.NewClient()
        if err != nil {
                log.Fatal(err)
        }
        c := newContext(cc, cl)

        grohl.Log(grohl.Data{"at": "leaderwait"})
        leaderWait, err := discoverd.RegisterAndStandby("flynn-controller-scheduler", ":"+os.Getenv("PORT"), nil)
        if err != nil {
                log.Fatal(err)
        }
        <-leaderWait
        grohl.Log(grohl.Data{"at": "leader"})

        // TODO: periodic full cluster sync for anti-entropy
        c.watchFormations(nil)
}

func newContext(cc controllerClient, cl clusterClient) *context {
        return &context{
                controllerClient: cc,
                clusterClient:    cl,
                formations:       NewFormations(),
                hosts:            newHostClients(),
                jobs:             newJobMap(),
        }
}

type context struct {
        controllerClient
        clusterClient
        formations *Formations

        hosts *hostClients
        jobs  *jobMap
        mtx   sync.RWMutex
}

:619
net.(*netFD).Read(0xc2100727e0, 0xc210081000, 0x1000, 0x1000, 0x0, ...)
	/usr/local/go/src/pkg/net/fd_unix.go:204 +0x2a0
net.(*conn).Read(0xc210000c90, 0xc210081000, 0x1000, 0x1000, 0x0, ...)
	/usr/local/go/src/pkg/net/net.go:122 +0xc5
bufio.(*Reader).fill(0xc21007d180)
	/usr/local/go/src/pkg/bufio/bufio.go:91 +0x110
bufio.(*Reader).Read(0xc21007d180, 0xc210073720, 0x1, 0x9, 0x1, ...)
	/usr/local/go/src/pkg/bufio/bufio.go:159 +0x1a4
io.ReadAtLeast(0x7f7f88034888, 0xc21007d180, 0xc210073720, 0x1, 0x9, ...)
	/usr/local/go/src/pkg/io/io.go:288 +0xf6
io.ReadFull(0x7f7f88034888, 0xc21007d180, 0xc210073720, 0x1, 0x9, ...)
	/usr/local/go/src/pkg/io/io.go:306 +0x71
encoding/gob.decodeUintReader(0x7f7f88034888, 0xc21007d180, 0xc210073720, 0x9, 0x9, ...)
	/usr/local/go/src/pkg/encoding/gob/decode.go:66 +0x98
encoding/gob.(*Decoder).recvMessage(0xc21000b3c0, 0xc210119000)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:73 +0x57
encoding/gob.(*Decoder).decodeTypeSequence(0xc21000b3c0, 0xc21006e000, 0x160)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:159 +0x49
encoding/gob.(*Decoder).DecodeValue(0xc21000b3c0, 0x6238a0, 0xc21006e000, 0x160, 0x0, ...)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:223 +0x12e
encoding/gob.(*Decoder).Decode(0xc21000b3c0, 0x6238a0, 0xc21006e000, 0x0, 0x1)
	/usr/local/go/src/pkg/encoding/gob/decoder.go:202 +0x1c5
github.com/flynn/rpcplus.(*gobClientCodec).ReadResponseHeader(0xc210077e40, 0xc21006e000, 0x120, 0x622e60)
	/tmp/godep/rev/05/df3d30b6e55f5b440c1a78e2f83cb822bc0d76/src/github.com/flynn/rpcplus/client.go:278 +0x3f
github.com/flynn/rpcplus.(*Client).input(0xc21007d1e0)
	/tmp/godep/rev/05/df3d30b6e55f5b440c1a78e2f83cb822bc0d76/src/github.com/flynn/rpcplus/client.go:138 +0xb8
created by github.com/flynn/rpcplus.NewClientWithCodec
	/tmp/godep/rev/05/df3d30b6e55f5b440c1a78e2f83cb822bc0d76/src/github.com/flynn/rpcplus/client.go:256 +0x91
vagrant@flynn:/vagrant/src/flynn-controller$
vagrant@flynn:/vagrant/src/flynn-controller$
vagrant@flynn:/vagrant/src/flynn-controller$ exit