raft panic on node restart

gistfile1.txt

I1022 10:18:36.467819 38605 acceptance/put_test.go:168  round 11: restarting nodes [2 1]
I1022 10:18:36.621232 38605 acceptance/put_test.go:173  restarting 0
I1022 10:18:36.908888 38605 acceptance/put_test.go:173  restarting 1
W1022 10:18:36.908977 38605 client/rpc_sender.go:93  failed to send RPC request Server.Batch: client is unhealthy
W1022 10:18:36.909913 38605 rpc/client.go:226  dial tcp 192.168.99.100:33538: getsockopt: connection refused
W1022 10:18:37.142376 38605 client/rpc_sender.go:93  failed to send RPC request Server.Batch: client is unhealthy
W1022 10:18:37.150573 38605 acceptance/put_test.go:181  monkey sleeping while cluster recovers...
W1022 10:18:37.150623 38605 client/rpc_sender.go:93  failed to send RPC request Server.Batch: client is unhealthy
W1022 10:18:37.151009 38605 rpc/client.go:226  dial tcp 192.168.99.100:33539: getsockopt: connection refused
I1022 10:18:37.280500 38605 acceptance/put_test.go:205  9854 (2604, 3469, 3781)
W1022 10:18:37.422904 38605 client/rpc_sender.go:93  failed to send RPC request Server.Batch: client is unhealthy
W1022 10:18:37.704927 38605 client/rpc_sender.go:93  failed to send RPC request Server.Batch: client is unhealthy
W1022 10:18:37.772442 38605 rpc/client.go:226  dial tcp 192.168.99.100:33538: getsockopt: connection refused
W1022 10:18:37.899638 38605 client/rpc_sender.go:93  failed to send RPC request Server.Batch: client is unhealthy
I1022 10:18:38.282828 38605 acceptance/put_test.go:205  9854 (2604, 3469, 3781)
W1022 10:18:38.663117 38605 client/rpc_sender.go:93  failed to send RPC request Server.Batch: client is unhealthy
I1022 10:18:39.284954 38605 acceptance/put_test.go:205  9964 (2604, 3578, 3782)
W1022 10:18:39.775259 38605 rpc/client.go:226  dial tcp 192.168.99.100:33538: getsockopt: connection refused
I1022 10:18:40.289111 38605 acceptance/put_test.go:205  10310 (2604, 3808, 3898)
W1022 10:18:40.524943 38605 client/rpc_sender.go:93  failed to send RPC request Server.Batch: client is unhealthy
I1022 10:18:42.122893 38605 acceptance/put_test.go:205  10623 (2604, 3946, 4073)
W1022 10:18:42.150915 38605 /usr/local/Cellar/go/1.5.1/libexec/src/runtime/asm_amd64.s:438  monkey out
W1022 10:18:44.110441 38605 rpc/client.go:226  dial tcp 192.168.99.100:33538: getsockopt: connection refused
W1022 10:18:44.934013 38605 client/rpc_sender.go:93  failed to send RPC request Server.Batch: client is unhealthy
I1022 10:18:44.934145 38605 acceptance/put_test.go:210  10623 166.5/sec
I1022 10:18:44.934339 38605 acceptance/localcluster/localcluster.go:483  asserted []
I1022 10:18:44.934362 38605 acceptance/localcluster/localcluster.go:501  stopping
--- FAIL: TestPutWithChaos (71.28s)
	localcluster.go:481: unexpected extra event &{0 die} (after [{0 die} {0 restart} {1 die} {1 restart}])
	put_test.go:195: failed to send RPC request Server.Batch: client is unhealthy
FAIL
exit status 1
FAIL	github.com/cockroachdb/cockroach/acceptance	71.310s
make: *** [acceptance] Error 1


panic on roach0:

I1022 14:18:37.128292 1 server/context.go:178  initialized 1 storage engine(s)
I1022 14:18:37.128400 1 cli/start.go:146  starting cockroach cluster
I1022 14:18:37.143595 1 storage/engine/rocksdb.go:103  opening rocksdb instance at "/data0"
I1022 14:18:37.167483 1 gossip/gossip.go:668  starting client to roach1.local:26257
W1022 14:18:37.177636 1 rpc/client.go:226  dial tcp 172.17.3.137:26257: getsockopt: connection refused
I1022 14:18:37.231345 1 server/node.go:286  initialized store store=1:1 ([ssd]=/data0): {Capacity:19507089408 Available:10216960000 RangeCount:0}
I1022 14:18:37.231717 1 server/node.go:213  node ID 0 initialized
I1022 14:18:37.233957 1 gossip/gossip.go:175  gossiping node descriptor node_id:1 address:<network_field:"tcp" address_field:"roach0.local:26257" > attrs:<>
I1022 14:18:37.236509 1 server/node.go:376  connecting to gossip network to verify cluster ID...
E1022 14:18:37.244554 1 raft/raft.go:780  [group 1] 1 state.commit 10062 is out of range [10, 9570]
panic: [group 1] 1 state.commit 10062 is out of range [10, 9570]

goroutine 25 [running]:
github.com/cockroachdb/cockroach/multiraft.(*raftLogger).Panicf(0xc82029b420, 0x1462500, 0x2b, 0xc8202ea8c0, 0x4, 0x4)
	/go/src/github.com/cockroachdb/cockroach/multiraft/raft.go:121 +0x1a2
github.com/coreos/etcd/raft.(*raft).loadState(0xc82028ad00, 0x12, 0x3, 0x274e, 0x0, 0x0, 0x0)
	/go/src/github.com/coreos/etcd/raft/raft.go:780 +0x2b3
github.com/coreos/etcd/raft.newRaft(0xc82027f570, 0xc82028c720)
	/go/src/github.com/coreos/etcd/raft/raft.go:205 +0x8f0
github.com/coreos/etcd/raft.(*multiNode).run(0xc82028c3c0)
	/go/src/github.com/coreos/etcd/raft/multinode.go:203 +0x423
created by github.com/coreos/etcd/raft.StartMultiNode
	/go/src/github.com/coreos/etcd/raft/multinode.go:71 +0x30d

goroutine 1 [runnable]:
github.com/cockroachdb/cockroach/server.(*Node).connectGossip(0xc8201e6480)
	/go/src/github.com/cockroachdb/cockroach/server/node.go:379 +0x98
github.com/cockroachdb/cockroach/server.(*Node).initStores(0xc8201e6480, 0xc8201fc870, 0x1, 0x1, 0xc82000f140, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/server/node.go:297 +0x3f2
github.com/cockroachdb/cockroach/server.(*Node).start(0xc8201e6480, 0xc82000d500, 0xc8201fc870, 0x1, 0x1, 0x0, 0x0, 0x0, 0xc82000f140, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/server/node.go:237 +0x2db
github.com/cockroachdb/cockroach/server.(*Server).Start(0xc8201fb4a0, 0xc82000f100, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/server/server.go:184 +0x445
github.com/cockroachdb/cockroach/cli.runStart(0x1b17ba0, 0xc82000d300, 0x0, 0x8)
	/go/src/github.com/cockroachdb/cockroach/cli/start.go:153 +0xad0
github.com/spf13/cobra.(*Command).execute(0x1b17ba0, 0xc82000d200, 0x8, 0x8, 0x0, 0x0)
	/go/src/github.com/spf13/cobra/command.go:551 +0x869
github.com/spf13/cobra.(*Command).Execute(0x1b15bc0, 0x0, 0x0)
	/go/src/github.com/spf13/cobra/command.go:634 +0x46a
github.com/cockroachdb/cockroach/cli.Run(0xc82000a0b0, 0x9, 0x9, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/cli/cli.go:87 +0x7c
main.main()
	/go/src/github.com/cockroachdb/cockroach/main.go:38 +0xe6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1696 +0x1

goroutine 6 [chan receive]:
github.com/cockroachdb/cockroach/util/log.(*loggingT).flushDaemon(0x1e58d80)
	/go/src/github.com/cockroachdb/cockroach/util/log/clog.go:1036 +0x64
created by github.com/cockroachdb/cockroach/util/log.init.1
	/go/src/github.com/cockroachdb/cockroach/util/log/clog.go:610 +0xbc

goroutine 8 [syscall]:
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 10 [select]:
github.com/cockroachdb/cockroach/rpc.(*RemoteClockMonitor).MonitorRemoteOffsets(0xc8202822c0, 0xc82000f140)
	/go/src/github.com/cockroachdb/cockroach/rpc/clock_offset.go:157 +0x741
github.com/cockroachdb/cockroach/server.NewServer.func1()
	/go/src/github.com/cockroachdb/cockroach/server/server.go:113 +0x2d
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc8202822e0)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 22 [select]:
github.com/cockroachdb/cockroach/util/retry.(*Retry).Next(0xc820051e10, 0x1)
	/go/src/github.com/cockroachdb/cockroach/util/retry/retry.go:109 +0x183
github.com/cockroachdb/cockroach/rpc.(*Client).runHeartbeat(0xc82000d600, 0x3b9aca00, 0x6fc23ac00, 0x4000000000000000, 0x0, 0x0, 0xc82000f140, 0xc82000f080)
	/go/src/github.com/cockroachdb/cockroach/rpc/client.go:221 +0x323
github.com/cockroachdb/cockroach/rpc.NewClient.func1()
	/go/src/github.com/cockroachdb/cockroach/rpc/client.go:138 +0x4f
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc820280b90)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 21 [select]:
github.com/cockroachdb/cockroach/gossip.(*client).start.func1()
	/go/src/github.com/cockroachdb/cockroach/gossip/client.go:65 +0x429
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc820279ec0)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 13 [select]:
github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1()
	/go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:215 +0x19b
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc8202823c0)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 14 [select]:
github.com/cockroachdb/cockroach/util.NewFeed.func1()
	/go/src/github.com/cockroachdb/cockroach/util/feed.go:62 +0x157
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc820275550)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 15 [select]:
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc8201fb7c0)
	/go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:214 +0xca5
github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm()
	/go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:201 +0x20
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc8202755a0)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 16 [select]:
github.com/cockroachdb/cockroach/gossip.(*server).start.func1()
	/go/src/github.com/cockroachdb/cockroach/gossip/server.go:145 +0x12e
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc8202825c0)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 18 [select]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1()
	/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:512 +0x1ec
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc8202825e0)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 19 [select]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1()
	/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:536 +0x1aa
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc820282600)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 20 [select]:
github.com/cockroachdb/cockroach/gossip.(*Gossip).maybeWarnAboutInit.func1()
	/go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:621 +0x483
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc820282620)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 24 [chan receive]:
github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc82012cb40)
	/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:119 +0x3a
created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open
	/go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:120 +0x489

goroutine 26 [select]:
github.com/coreos/etcd/raft.(*multiNode).CreateGroup(0xc82028c3c0, 0x1, 0xc82027f570, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/coreos/etcd/raft/multinode.go:354 +0x1e6
github.com/cockroachdb/cockroach/multiraft.(*state).createGroup(0xc820281040, 0x1, 0xc800000001, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/multiraft/multiraft.go:874 +0xc68
github.com/cockroachdb/cockroach/multiraft.(*state).start.func1()
	/go/src/github.com/cockroachdb/cockroach/multiraft/multiraft.go:585 +0x9f1
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc82029a6e0)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 27 [chan receive]:
github.com/cockroachdb/cockroach/storage.(*Store).proposeRaftCommandImpl(0xc8200e05a0, 0xc82029b320, 0x10, 0x1, 0x100000001, 0x1, 0x0, 0x0, 0x140f87faef83f2c2, 0x19cbca5c00389ebe, ...)
	/go/src/github.com/cockroachdb/cockroach/storage/store.go:1403 +0x197
github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1()
	/go/src/github.com/cockroachdb/cockroach/storage/store.go:1503 +0xf76
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc82029a6f0)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 28 [select]:
github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1()
	/go/src/github.com/cockroachdb/cockroach/storage/store.go:576 +0x3bf
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc820282f20)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 29 [chan receive]:
github.com/cockroachdb/cockroach/storage.(*Store).ProposeRaftCommand(0xc8200e05a0, 0xc82029b320, 0x10, 0x1, 0x100000001, 0x1, 0x0, 0x0, 0x140f87faef83f2c2, 0x19cbca5c00389ebe, ...)
	/go/src/github.com/cockroachdb/cockroach/storage/store.go:1397 +0xe1
github.com/cockroachdb/cockroach/storage.(*Replica).proposeRaftCommand(0xc820164240, 0x7f8c1aa1de38, 0xc820299530, 0x0, 0x0, 0x140f87faef83f2c2, 0x19cbca5c00389ebe, 0x0, 0x0, 0x1, ...)
	/go/src/github.com/cockroachdb/cockroach/storage/replica.go:892 +0x7b2
github.com/cockroachdb/cockroach/storage.(*Replica).requestLeaderLease(0xc820164240, 0x140f87faef83e566, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/storage/replica.go:375 +0x5d0
github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLeaderLease(0xc820164240, 0x0, 0x140f87faef83e566, 0xc800000000, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/storage/replica.go:413 +0x329
github.com/cockroachdb/cockroach/storage.(*Replica).getLeaseForGossip.func1()
	/go/src/github.com/cockroachdb/cockroach/storage/replica.go:1210 +0xdd
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc82000f140, 0xc82004bd70, 0xc8202ea700)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:115 +0x234
github.com/cockroachdb/cockroach/storage.(*Replica).getLeaseForGossip(0xc820164240, 0x7f8c1aa1de38, 0xc820299470, 0x7f8c1aa1de38, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/storage/replica.go:1223 +0x128
github.com/cockroachdb/cockroach/storage.(*Store).maybeGossipSystemConfig(0xc8200e05a0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/storage/store.go:630 +0xcc
github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2()
	/go/src/github.com/cockroachdb/cockroach/storage/store.go:588 +0x58
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc820282f40)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 30 [runnable]:
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc82029a760)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:86
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 32 [select]:
github.com/cockroachdb/cockroach/multiraft.(*writeTask).start.func1()
	/go/src/github.com/cockroachdb/cockroach/multiraft/storage.go:188 +0xca0
github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc82000f140, 0xc820283680)
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:88 +0x52
created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:89 +0x62

goroutine 33 [runnable]:
github.com/cockroachdb/cockroach/gossip.(*infoStore).processCallbacks.func1(0x0, 0x0, 0x0, 0x1372530, 0xa, 0xc8202993b0, 0x24, 0x30)
	/go/src/github.com/cockroachdb/cockroach/gossip/infostore.go:225
created by github.com/cockroachdb/cockroach/gossip.(*infoStore).processCallbacks
	/go/src/github.com/cockroachdb/cockroach/gossip/infostore.go:229 +0x1e9

goroutine 34 [runnable]:
github.com/cockroachdb/cockroach/gossip.(*infoStore).processCallbacks.func1(0x0, 0x0, 0x0, 0x137de90, 0x8, 0xc8202993e0, 0x24, 0x30)
	/go/src/github.com/cockroachdb/cockroach/gossip/infostore.go:225
created by github.com/cockroachdb/cockroach/gossip.(*infoStore).processCallbacks
	/go/src/github.com/cockroachdb/cockroach/gossip/infostore.go:229 +0x1e9
✔ ~/go/src/github.com/cockroachdb/cockroach [monkey_test|✚ 4]
10:26 $