Created
September 8, 2016 20:37
-
-
Save tamird/64daf12e5349f2d6de42d171edda9612 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| === RUN TestWaitForFullReplication | |
| I160906 11:36:32.496285 927 storage/engine/rocksdb.go:354 opening in memory rocksdb instance | |
| I160906 11:36:32.524863 927 gossip/gossip.go:230 [n?] initial resolvers: [] | |
| W160906 11:36:32.525013 927 gossip/gossip.go:1041 [n?] no resolvers found; use --join to specify a connected node | |
| W160906 11:36:32.529079 927 server/status/runtime.go:109 Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006" | |
| I160906 11:36:32.544720 927 server/node.go:413 [n?] store store=0:0 not bootstrapped | |
| I160906 11:36:32.573024 980 storage/replica.go:2237 [s1,r1{/Min-/Max}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:32.573772 980 storage/replica.go:2272 [s1,r1{/Min-/Max}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:32.574558 980 storage/replica_trigger.go:308 [s1,r1{/Min-/Max}] new range lease replica {1 1 1} 1970-01-01 00:00:00 +0000 UTC 409211h36m41.571292356s following replica {0 0 0} 1970-01-01 00:00:00 +0000 UTC 0s [physicalTime=2016-09-06 11:36:32.574355943 +0000 UTC] | |
| I160906 11:36:32.579681 927 util/stop/stopper.go:342 stop has been called, stopping or quiescing all running tasks | |
| I160906 11:36:32.580028 927 server/node.go:345 [n?] **** cluster {9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124} has been created | |
| I160906 11:36:32.580114 927 server/node.go:346 [n?] **** add additional nodes by specifying --join=127.0.0.1:56887 | |
| I160906 11:36:32.581725 927 gossip/gossip.go:264 [n?] NodeID set to 1 | |
| I160906 11:36:32.587474 974 storage/replica.go:2845 [n?,s1,r1{/Min-/Max}] gossiping cluster id {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} from store 1, range 1 | |
| I160906 11:36:32.587921 974 storage/replica.go:2867 [n?,s1,r1{/Min-/Max}] gossiping sentinel from store 1, range 1 | |
| I160906 11:36:32.588281 974 storage/replica.go:2874 [n?,s1,r1{/Min-/Max}] gossiping first range from store 1, range 1: [{1 1 1}] | |
| I160906 11:36:32.588733 987 kv/dist_sender.go:223 [n?] gossipped first range descriptor: [{NodeID:1 StoreID:1 ReplicaID:1}] | |
| I160906 11:36:32.590792 990 storage/split_queue.go:99 splitting store=1:1 1{/Min-/Max} at keys [/Table/11/0 /Table/12/0 /Table/13/0 /Table/14/0] | |
| I160906 11:36:32.591284 990 kv/replica_slice.go:56 node 1 is not gossiped: unable to lookup descriptor for node 1 | |
| W160906 11:36:32.591562 990 kv/dist_sender.go:699 no replica node addresses available via gossip | |
| I160906 11:36:32.592257 927 server/node.go:426 [n?] initialized store store=1:1: {Capacity:536870912 Available:536870912 RangeCount:0} | |
| I160906 11:36:32.592440 927 server/node.go:318 [n?] node ID 1 initialized | |
| I160906 11:36:32.592675 927 gossip/gossip.go:278 [n?] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:56887" > attrs:<> | |
| I160906 11:36:32.594659 927 storage/stores.go:313 read 0 node addresses from persistent storage | |
| I160906 11:36:32.594886 927 server/node.go:554 [n?] connecting to gossip network to verify cluster ID... | |
| I160906 11:36:32.595022 927 server/node.go:575 [n?] node connected via gossip and verified as part of cluster {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} | |
| I160906 11:36:32.595525 927 server/node.go:364 [n?] node=1: started with [[]=] engine(s) and attributes [] | |
| I160906 11:36:32.596143 1013 sql/event_log.go:109 [n1] Event: "node_join", target: 1, info: {Descriptor:{NodeID:1 Address:{NetworkField:tcp AddressField:127.0.0.1:56887} Attrs:} ClusterID:{UUID:9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124} StartedAt:1473161792595072036} | |
| I160906 11:36:32.596219 927 server/server.go:490 [n1] starting https server at 127.0.0.1:38660 | |
| I160906 11:36:32.596288 927 server/server.go:491 [n1] starting grpc/postgres server at 127.0.0.1:56887 | |
| I160906 11:36:32.596538 927 rpc/context.go:203 dialing 127.0.0.1:56887 | |
| I160906 11:36:32.707182 983 storage/replica.go:2237 [n1,s1,r1{/Min-/Max}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:32.716966 983 storage/replica.go:2272 [n1,s1,r1{/Min-/Max}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:32.744953 990 storage/replica_command.go:2190 [n1,s1,r1{/Min-/Max}] initiating a split of this range at key /Table/11 | |
| I160906 11:36:32.777905 982 storage/replica_command.go:1357 [n1,s1,r1{/Min-/Max}] <nil> pushed a304fc8e: priority | |
| I160906 11:36:32.793923 1001 rpc/clock_offset.go:147 0 of 0 nodes are within the maximum offset of 50ms | |
| I160906 11:36:32.799988 990 storage/log.go:85 Range Event: "split", range: 1, info: {"UpdatedDesc":{"range_id":1,"end_key":"kw==","replicas":[{"node_id":1,"store_id":1,"replica_id":1}],"next_replica_id":2},"NewDesc":{"range_id":2,"start_key":"kw==","end_key":"//8=","replicas":[{"node_id":1,"store_id":1,"replica_id":1}],"next_replica_id":2}} | |
| W160906 11:36:32.837408 927 storage/store.go:2089 [n1,s1] conflicting intents on [/Meta2/Table/11 /Meta2/Max]: resolved? false | |
| I160906 11:36:32.840781 990 kv/dist_sender.go:1036 application error: retry txn "storage/replica_command.go:2236 (*Replica).AdminSplit" id=a304fc8e key=/Local/Range/""/RangeDescriptor rw=true pri=0.02264497 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161792.772731588,1 orig=1473161792.751635878,0 max=1473161792.751635878,0 wto=false rop=false | |
| I160906 11:36:32.841050 990 kv/dist_sender.go:771 reply error EndTransaction [/Local/Range/""/RangeDescriptor,/Min): retry txn "storage/replica_command.go:2236 (*Replica).AdminSplit" id=a304fc8e key=/Local/Range/""/RangeDescriptor rw=true pri=0.02264497 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161792.772731588,1 orig=1473161792.751635878,0 max=1473161792.751635878,0 wto=false rop=false | |
| I160906 11:36:32.841589 990 internal/client/db.go:498 failed batch: retry txn "storage/replica_command.go:2236 (*Replica).AdminSplit" id=a304fc8e key=/Local/Range/""/RangeDescriptor rw=true pri=0.02264497 iso=SERIALIZABLE stat=PENDING epo=1 ts=1473161792.772731588,1 orig=1473161792.772731588,1 max=1473161792.801635878,0 wto=false rop=false | |
| I160906 11:36:32.846454 990 storage/log.go:85 Range Event: "split", range: 1, info: {"UpdatedDesc":{"range_id":1,"end_key":"kw==","replicas":[{"node_id":1,"store_id":1,"replica_id":1}],"next_replica_id":2},"NewDesc":{"range_id":2,"start_key":"kw==","end_key":"//8=","replicas":[{"node_id":1,"store_id":1,"replica_id":1}],"next_replica_id":2}} | |
| I160906 11:36:32.896684 984 storage/queue.go:293 [replicate] store=1:1 2{/Table/11-/Max}: split needed; not adding | |
| E160906 11:36:32.914638 992 storage/queue.go:517 [replicate] (purgatory) on store=1:1 1{/Min-/Table/11}: 0 of 0 stores with all attributes matching []; likely not enough nodes in cluster | |
| I160906 11:36:32.917755 985 storage/replica.go:2237 [n1,s1,r2{/Table/11-/Max}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:32.919865 985 storage/replica.go:2272 [n1,s1,r2{/Table/11-/Max}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:32.922008 990 kv/dist_sender.go:1036 application error: key range /Table/12/0-/Min outside of bounds of range /Min-/Table/11 | |
| I160906 11:36:32.922512 990 kv/dist_sender.go:771 reply error AdminSplit [/Table/12/0,/Min): key range /Table/12/0-/Min outside of bounds of range /Min-/Table/11 | |
| W160906 11:36:32.923402 990 kv/dist_sender.go:816 key range /Table/12/0-/Min outside of bounds of range /Min-/Table/11 | |
| E160906 11:36:32.924379 1022 storage/queue.go:517 [replicate] (purgatory) on store=1:1 1{/Min-/Table/11}: 0 of 0 stores with all attributes matching []; likely not enough nodes in cluster | |
| I160906 11:36:32.929400 990 storage/replica_command.go:2190 [n1,s1,r2{/Table/11-/Max}] initiating a split of this range at key /Table/12 | |
| W160906 11:36:32.941925 927 storage/store.go:2089 [n1,s1] conflicting intents on [/Meta2/Table/11 /Meta2/Max]: resolved? false | |
| I160906 11:36:32.983363 990 storage/log.go:85 Range Event: "split", range: 2, info: {"UpdatedDesc":{"range_id":2,"start_key":"kw==","end_key":"lA==","replicas":[{"node_id":1,"store_id":1,"replica_id":1}],"next_replica_id":2},"NewDesc":{"range_id":3,"start_key":"lA==","end_key":"//8=","replicas":[{"node_id":1,"store_id":1,"replica_id":1}],"next_replica_id":2}} | |
| I160906 11:36:33.028058 984 storage/queue.go:293 [replicate] store=1:1 3{/Table/12-/Max}: split needed; not adding | |
| E160906 11:36:33.045392 992 storage/queue.go:517 [replicate] (purgatory) on store=1:1 2{/Table/11-/Table/12}: 0 of 0 stores with all attributes matching []; likely not enough nodes in cluster | |
| I160906 11:36:33.049208 990 kv/dist_sender.go:1036 application error: key range /Table/13/0-/Min outside of bounds of range /Table/11-/Table/12 | |
| I160906 11:36:33.056230 990 kv/dist_sender.go:771 reply error AdminSplit [/Table/13/0,/Min): key range /Table/13/0-/Min outside of bounds of range /Table/11-/Table/12 | |
| I160906 11:36:33.058519 985 storage/replica.go:2237 [n1,s1,r3{/Table/12-/Max}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:33.059030 985 storage/replica.go:2272 [n1,s1,r3{/Table/12-/Max}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| W160906 11:36:33.059322 990 kv/dist_sender.go:816 key range /Table/13/0-/Min outside of bounds of range /Table/11-/Table/12 | |
| I160906 11:36:33.060399 990 storage/replica_command.go:2190 [n1,s1,r3{/Table/12-/Max}] initiating a split of this range at key /Table/13 | |
| I160906 11:36:33.068329 1028 kv/dist_sender.go:1036 application error: end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| I160906 11:36:33.091540 1028 kv/dist_sender.go:771 reply error GC [/Local/Range/"\x93"/RangeDescriptor,/Local/Range/"\x93"/"rdsc\x00"): end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| I160906 11:36:33.091794 1028 internal/client/db.go:498 failed batch: end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| W160906 11:36:33.091880 1028 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| W160906 11:36:33.127150 927 storage/store.go:2089 [n1,s1] conflicting intents on [/Meta2/Table/12 /Meta2/Max]: resolved? false | |
| I160906 11:36:33.139427 990 storage/log.go:85 Range Event: "split", range: 3, info: {"UpdatedDesc":{"range_id":3,"start_key":"lA==","end_key":"lQ==","replicas":[{"node_id":1,"store_id":1,"replica_id":1}],"next_replica_id":2},"NewDesc":{"range_id":4,"start_key":"lQ==","end_key":"//8=","replicas":[{"node_id":1,"store_id":1,"replica_id":1}],"next_replica_id":2}} | |
| I160906 11:36:33.147508 984 storage/replica_command.go:1357 [n1,s1,r3{/Table/12-/Max}] <nil> failed to push 3ed4865d: priority | |
| I160906 11:36:33.148013 984 storage/replica_command.go:1363 [n1,s1,r3{/Table/12-/Max}] failed to push "storage/replica_command.go:2236 (*Replica).AdminSplit" id=3ed4865d key=/Local/Range/"\x94"/RangeDescriptor rw=true pri=0.01360378 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161793.060754025,0 orig=1473161793.060754025,0 max=1473161793.060754025,0 wto=false rop=false | |
| I160906 11:36:33.164707 927 kv/dist_sender.go:1036 application error: failed to push "storage/replica_command.go:2236 (*Replica).AdminSplit" id=3ed4865d key=/Local/Range/"\x94"/RangeDescriptor rw=true pri=0.01360378 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161793.060754025,0 orig=1473161793.060754025,0 max=1473161793.060754025,0 wto=false rop=false | |
| I160906 11:36:33.164989 927 kv/dist_sender.go:771 reply error PushTxn [/Local/Range/"\x94"/RangeDescriptor,/Min), PushTxn [/Local/Range/"\x94"/RangeDescriptor,/Min): failed to push "storage/replica_command.go:2236 (*Replica).AdminSplit" id=3ed4865d key=/Local/Range/"\x94"/RangeDescriptor rw=true pri=0.01360378 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161793.060754025,0 orig=1473161793.060754025,0 max=1473161793.060754025,0 wto=false rop=false | |
| I160906 11:36:33.165152 927 internal/client/db.go:498 failed batch: failed to push "storage/replica_command.go:2236 (*Replica).AdminSplit" id=3ed4865d key=/Local/Range/"\x94"/RangeDescriptor rw=true pri=0.01360378 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161793.060754025,0 orig=1473161793.060754025,0 max=1473161793.060754025,0 wto=false rop=false | |
| I160906 11:36:33.165410 927 storage/intent_resolver.go:102 [n1,s1] on Scan: failed to push "storage/replica_command.go:2236 (*Replica).AdminSplit" id=3ed4865d key=/Local/Range/"\x94"/RangeDescriptor rw=true pri=0.01360378 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161793.060754025,0 orig=1473161793.060754025,0 max=1473161793.060754025,0 wto=false rop=false | |
| W160906 11:36:33.176398 927 storage/store.go:2094 [n1,s1] conflicting intents on [/Meta2/Table/13 /Meta2/Max]: resolved? false | |
| I160906 11:36:33.200287 984 storage/queue.go:293 [replicate] store=1:1 4{/Table/13-/Max}: split needed; not adding | |
| E160906 11:36:33.236452 992 storage/queue.go:517 [replicate] (purgatory) on store=1:1 3{/Table/12-/Table/13}: 0 of 0 stores with all attributes matching []; likely not enough nodes in cluster | |
| I160906 11:36:33.238717 990 kv/dist_sender.go:1036 application error: key range /Table/14/0-/Min outside of bounds of range /Table/12-/Table/13 | |
| I160906 11:36:33.238863 990 kv/dist_sender.go:771 reply error AdminSplit [/Table/14/0,/Min): key range /Table/14/0-/Min outside of bounds of range /Table/12-/Table/13 | |
| W160906 11:36:33.239202 990 kv/dist_sender.go:816 key range /Table/14/0-/Min outside of bounds of range /Table/12-/Table/13 | |
| I160906 11:36:33.240064 990 storage/replica_command.go:2190 [n1,s1,r4{/Table/13-/Max}] initiating a split of this range at key /Table/14 | |
| I160906 11:36:33.243666 983 storage/replica.go:2237 [n1,s1,r4{/Table/13-/Max}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:33.244623 983 storage/replica.go:2272 [n1,s1,r4{/Table/13-/Max}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:33.246064 1030 kv/dist_sender.go:1036 application error: end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| I160906 11:36:33.246289 1030 kv/dist_sender.go:771 reply error GC [/Local/Range/"\x94"/RangeDescriptor,/Local/Range/"\x94"/"rdsc\x00"): end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| I160906 11:36:33.246444 1030 internal/client/db.go:498 failed batch: end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| W160906 11:36:33.246517 1030 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| I160906 11:36:33.277055 985 storage/replica_command.go:1357 [n1,s1,r4{/Table/13-/Max}] <nil> failed to push 1c4df1bf: priority | |
| I160906 11:36:33.277599 985 storage/replica_command.go:1363 [n1,s1,r4{/Table/13-/Max}] failed to push "storage/replica_command.go:2236 (*Replica).AdminSplit" id=1c4df1bf key=/Local/Range/"\x95"/RangeDescriptor rw=true pri=0.05077808 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161793.240503813,0 orig=1473161793.240503813,0 max=1473161793.240503813,0 wto=false rop=false | |
| I160906 11:36:33.278809 927 kv/dist_sender.go:1036 application error: failed to push "storage/replica_command.go:2236 (*Replica).AdminSplit" id=1c4df1bf key=/Local/Range/"\x95"/RangeDescriptor rw=true pri=0.05077808 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161793.240503813,0 orig=1473161793.240503813,0 max=1473161793.240503813,0 wto=false rop=false | |
| I160906 11:36:33.279053 927 kv/dist_sender.go:771 reply error PushTxn [/Local/Range/"\x95"/RangeDescriptor,/Min), PushTxn [/Local/Range/"\x95"/RangeDescriptor,/Min): failed to push "storage/replica_command.go:2236 (*Replica).AdminSplit" id=1c4df1bf key=/Local/Range/"\x95"/RangeDescriptor rw=true pri=0.05077808 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161793.240503813,0 orig=1473161793.240503813,0 max=1473161793.240503813,0 wto=false rop=false | |
| I160906 11:36:33.279404 990 storage/log.go:85 Range Event: "split", range: 4, info: {"UpdatedDesc":{"range_id":4,"start_key":"lQ==","end_key":"lg==","replicas":[{"node_id":1,"store_id":1,"replica_id":1}],"next_replica_id":2},"NewDesc":{"range_id":5,"start_key":"lg==","end_key":"//8=","replicas":[{"node_id":1,"store_id":1,"replica_id":1}],"next_replica_id":2}} | |
| I160906 11:36:33.298578 927 internal/client/db.go:498 failed batch: failed to push "storage/replica_command.go:2236 (*Replica).AdminSplit" id=1c4df1bf key=/Local/Range/"\x95"/RangeDescriptor rw=true pri=0.05077808 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161793.240503813,0 orig=1473161793.240503813,0 max=1473161793.240503813,0 wto=false rop=false | |
| I160906 11:36:33.312428 927 storage/intent_resolver.go:102 [n1,s1] on Scan: failed to push "storage/replica_command.go:2236 (*Replica).AdminSplit" id=1c4df1bf key=/Local/Range/"\x95"/RangeDescriptor rw=true pri=0.05077808 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161793.240503813,0 orig=1473161793.240503813,0 max=1473161793.240503813,0 wto=false rop=false | |
| W160906 11:36:33.312682 927 storage/store.go:2094 [n1,s1] conflicting intents on [/Meta2/Table/14 /Meta2/Max]: resolved? false | |
| I160906 11:36:33.347029 982 storage/replica.go:2237 [n1,s1,r5{/Table/14-/Max}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:33.351020 982 storage/replica.go:2272 [n1,s1,r5{/Table/14-/Max}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:33.358726 1023 kv/dist_sender.go:1036 application error: end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| I160906 11:36:33.358960 1023 kv/dist_sender.go:771 reply error GC [/Local/Range/"\x95"/RangeDescriptor,/Local/Range/"\x95"/"rdsc\x00"): end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| I160906 11:36:33.359136 1023 internal/client/db.go:498 failed batch: end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| W160906 11:36:33.359209 1023 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| E160906 11:36:33.359695 992 storage/queue.go:517 [replicate] (purgatory) on store=1:1 4{/Table/13-/Table/14}: 0 of 0 stores with all attributes matching []; likely not enough nodes in cluster | |
| E160906 11:36:33.360803 992 storage/queue.go:517 [replicate] (purgatory) on store=1:1 5{/Table/14-/Max}: 0 of 0 stores with all attributes matching []; likely not enough nodes in cluster | |
| I160906 11:36:33.365991 927 storage/engine/rocksdb.go:354 opening in memory rocksdb instance | |
| I160906 11:36:33.420694 927 gossip/gossip.go:230 [n?] initial resolvers: [%!s(*resolver.socketResolver=&{tcp 127.0.0.1:56887})] | |
| W160906 11:36:33.420846 927 gossip/gossip.go:1043 [n?] no incoming or outgoing connections | |
| W160906 11:36:33.424698 927 server/status/runtime.go:109 Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006" | |
| I160906 11:36:33.429533 927 server/node.go:413 [n?] store store=0:0 not bootstrapped | |
| I160906 11:36:33.429632 927 storage/stores.go:313 read 0 node addresses from persistent storage | |
| I160906 11:36:33.429758 927 server/node.go:554 [n?] connecting to gossip network to verify cluster ID... | |
| I160906 11:36:33.439633 1053 rpc/context.go:203 dialing 127.0.0.1:56887 | |
| I160906 11:36:33.537627 1053 gossip/client.go:125 [n?] node 0: started gossip client to 127.0.0.1:56887 | |
| I160906 11:36:33.562914 1058 rpc/clock_offset.go:176 uncertain remote offset off=42.31764ms, err=43.465044ms, at=2016-09-06 11:36:33.56277344 +0000 UTC for maximum offset 50ms, treating as healthy | |
| I160906 11:36:33.566170 1058 rpc/clock_offset.go:147 1 of 1 nodes are within the maximum offset of 50ms | |
| I160906 11:36:33.567765 1063 gossip/server.go:162 [n1] node 1: returning 6 info(s) to node 0: [system-db cluster-id sentinel first-range node:1 store:1] | |
| I160906 11:36:33.569844 1076 gossip/server.go:274 [n1] node 1: received gossip from unknown node | |
| I160906 11:36:33.570786 1076 gossip/server.go:289 [n1] node 1: received [] from node 0 (0 fresh) | |
| I160906 11:36:33.578061 1074 gossip/client.go:220 [n?] node 0: received [system-db cluster-id first-range node:1 sentinel store:1] from node 1 (6 fresh) | |
| I160906 11:36:33.578780 1082 kv/dist_sender.go:223 [n?] gossipped first range descriptor: [{NodeID:1 StoreID:1 ReplicaID:1}] | |
| I160906 11:36:33.579125 1082 storage/stores.go:328 wrote 1 node addresses to persistent storage | |
| I160906 11:36:33.581323 927 server/node.go:575 [n?] node connected via gossip and verified as part of cluster {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} | |
| I160906 11:36:33.600815 927 kv/dist_sender.go:344 [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping | |
| I160906 11:36:33.609664 927 server/node.go:312 [n?] new node allocated ID 2 | |
| I160906 11:36:33.610024 927 gossip/gossip.go:264 [n?] NodeID set to 2 | |
| I160906 11:36:33.610718 927 gossip/gossip.go:278 [n?] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:41417" > attrs:<> | |
| I160906 11:36:33.612317 927 server/node.go:364 [n?] node=2: started with [[]=] engine(s) and attributes [] | |
| I160906 11:36:33.613324 927 server/server.go:490 [n2] starting https server at 127.0.0.1:36224 | |
| I160906 11:36:33.615558 927 server/server.go:491 [n2] starting grpc/postgres server at 127.0.0.1:41417 | |
| I160906 11:36:33.615791 927 rpc/context.go:203 dialing 127.0.0.1:41417 | |
| I160906 11:36:33.616646 1053 gossip/client.go:189 [n2] node 2: sending [node:2] | |
| I160906 11:36:33.617826 1093 sql/event_log.go:109 [n2] Event: "node_join", target: 2, info: {Descriptor:{NodeID:2 Address:{NetworkField:tcp AddressField:127.0.0.1:41417} Attrs:} ClusterID:{UUID:9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124} StartedAt:1473161793611159410} | |
| I160906 11:36:33.640478 1076 gossip/server.go:289 [n1] node 1: received [node:2] from node 2 (1 fresh) | |
| I160906 11:36:33.641525 1102 storage/stores.go:328 wrote 1 node addresses to persistent storage | |
| I160906 11:36:33.755790 927 storage/engine/rocksdb.go:354 opening in memory rocksdb instance | |
| I160906 11:36:33.802890 1090 gossip/gossip.go:264 [n2] NodeID set to 2 | |
| I160906 11:36:33.827967 1090 server/node.go:535 [n2] bootstrapped store store=2:2 | |
| I160906 11:36:33.839046 927 gossip/gossip.go:230 [n?] initial resolvers: [%!s(*resolver.socketResolver=&{tcp 127.0.0.1:56887})] | |
| W160906 11:36:33.839223 927 gossip/gossip.go:1043 [n?] no incoming or outgoing connections | |
| W160906 11:36:33.846122 927 server/status/runtime.go:109 Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006" | |
| I160906 11:36:33.855907 927 server/node.go:413 [n?] store store=0:0 not bootstrapped | |
| I160906 11:36:33.898139 927 storage/stores.go:313 read 0 node addresses from persistent storage | |
| I160906 11:36:33.901867 927 server/node.go:554 [n?] connecting to gossip network to verify cluster ID... | |
| I160906 11:36:33.918341 1053 gossip/client.go:189 [n2] node 2: sending [store:2] | |
| I160906 11:36:33.921158 1163 rpc/context.go:203 dialing 127.0.0.1:56887 | |
| I160906 11:36:33.926472 1104 rpc/clock_offset.go:176 uncertain remote offset off=42.31764ms, err=43.465044ms, at=2016-09-06 11:36:33.56277344 +0000 UTC for maximum offset 50ms, treating as healthy | |
| I160906 11:36:33.926556 1104 rpc/clock_offset.go:147 1 of 1 nodes are within the maximum offset of 50ms | |
| I160906 11:36:33.946363 1076 gossip/server.go:289 [n1] node 1: received [store:2] from node 2 (1 fresh) | |
| I160906 11:36:34.001595 1022 storage/replicate_queue.go:154 store=1:1 1{/Min-/Table/11}: adding replica to {NodeID:2 StoreID:2 ReplicaID:0} due to under-replication | |
| I160906 11:36:34.001774 1022 rpc/context.go:203 dialing 127.0.0.1:41417 | |
| I160906 11:36:34.082100 1163 gossip/client.go:125 [n?] node 0: started gossip client to 127.0.0.1:56887 | |
| I160906 11:36:34.106782 1177 gossip/server.go:162 [n1] node 1: returning 8 info(s) to node 0: [first-range node:1 store:1 node:2 store:2 system-db cluster-id sentinel] | |
| I160906 11:36:34.172495 1179 gossip/server.go:274 [n1] node 1: received gossip from unknown node | |
| I160906 11:36:34.172844 1179 gossip/server.go:289 [n1] node 1: received [] from node 0 (0 fresh) | |
| I160906 11:36:34.236110 1176 gossip/client.go:220 [n?] node 0: received [sentinel store:1 store:2 system-db cluster-id first-range node:1 node:2] from node 1 (8 fresh) | |
| I160906 11:36:34.241612 1202 kv/dist_sender.go:223 [n?] gossipped first range descriptor: [{NodeID:1 StoreID:1 ReplicaID:1}] | |
| I160906 11:36:34.241961 1202 storage/stores.go:328 wrote 1 node addresses to persistent storage | |
| I160906 11:36:34.242211 1202 storage/stores.go:328 wrote 2 node addresses to persistent storage | |
| I160906 11:36:34.245546 1148 rpc/clock_offset.go:147 0 of 0 nodes are within the maximum offset of 50ms | |
| I160906 11:36:34.245723 927 server/node.go:575 [n?] node connected via gossip and verified as part of cluster {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} | |
| I160906 11:36:34.256515 1211 storage/reservation.go:197 new reservation added: header:<node_id:2 store_id:2 > from_node_id:1 from_store_id:1 range_id:1 range_size:43361 | |
| I160906 11:36:34.273333 1188 rpc/clock_offset.go:147 0 of 0 nodes are within the maximum offset of 50ms | |
| I160906 11:36:34.273965 1180 storage/replica_raftstorage.go:488 generated snapshot 9da76f6b-904a-4258-8d15-760ff2b50c59 for range 1 at index 43 in 237.451µs. 0 log entries | |
| I160906 11:36:34.328539 927 kv/dist_sender.go:344 [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping | |
| I160906 11:36:34.340907 927 server/node.go:312 [n?] new node allocated ID 3 | |
| I160906 11:36:34.341046 927 gossip/gossip.go:264 [n?] NodeID set to 3 | |
| I160906 11:36:34.341266 927 gossip/gossip.go:278 [n?] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:59236" > attrs:<> | |
| I160906 11:36:34.342102 927 server/node.go:364 [n?] node=3: started with [[]=] engine(s) and attributes [] | |
| I160906 11:36:34.342624 927 server/server.go:490 [n3] starting https server at 127.0.0.1:42440 | |
| I160906 11:36:34.348168 927 server/server.go:491 [n3] starting grpc/postgres server at 127.0.0.1:59236 | |
| I160906 11:36:34.348407 927 rpc/context.go:203 dialing 127.0.0.1:59236 | |
| I160906 11:36:34.357607 1163 gossip/client.go:189 [n3] node 3: sending [node:3] | |
| I160906 11:36:34.358240 1222 sql/event_log.go:109 [n3] Event: "node_join", target: 3, info: {Descriptor:{NodeID:3 Address:{NetworkField:tcp AddressField:127.0.0.1:59236} Attrs:} ClusterID:{UUID:9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124} StartedAt:1473161794341720637} | |
| I160906 11:36:34.386807 1179 gossip/server.go:289 [n1] node 1: received [node:3] from node 3 (1 fresh) | |
| I160906 11:36:34.387737 1214 storage/stores.go:328 wrote 2 node addresses to persistent storage | |
| I160906 11:36:34.419152 1063 gossip/server.go:162 [n1] node 1: returning 1 info(s) to node 2: [node:3] | |
| I160906 11:36:34.421089 1022 storage/store.go:2634 streamed snapshot: kv pairs: 418, log entries: 33 | |
| I160906 11:36:34.421735 1183 storage/replica_raftstorage.go:618 store=2:2 1{/Min-/Min}: with replicaID [?], applying preemptive snapshot at index 43 (id=9da76f6b, encoded size=16, 1 rocksdb batches, 33 log entries) | |
| I160906 11:36:34.474533 1183 storage/replica_raftstorage.go:621 store=2:2 1{/Min-/Table/11}: with replicaID [?], applied preemptive snapshot in 0.052s | |
| I160906 11:36:34.476671 1074 gossip/client.go:220 [n2] node 2: received [node:3] from node 1 (1 fresh) | |
| I160906 11:36:34.477257 1240 storage/stores.go:328 wrote 2 node addresses to persistent storage | |
| I160906 11:36:34.503698 1219 gossip/gossip.go:264 [n3] NodeID set to 3 | |
| I160906 11:36:34.511380 1022 storage/replica_command.go:3058 change replicas of 1: read existing descriptor range_id:1 start_key:"" end_key:"\223" replicas:<node_id:1 store_id:1 replica_id:1 > next_replica_id:2 | |
| I160906 11:36:34.514085 1219 server/node.go:535 [n3] bootstrapped store store=3:3 | |
| I160906 11:36:34.542717 985 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: BeginTransaction [/Local/Range/""/RangeDescriptor,/Min), ConditionalPut [/Local/Range/""/RangeDescriptor,/Min), Put [/Meta2/Table/11,/Min), Put [/Meta1/Max,/Min), pErr=<nil> | |
| I160906 11:36:34.598632 1163 gossip/client.go:189 [n3] node 3: sending [store:3] | |
| I160906 11:36:34.601226 1022 storage/log.go:85 Range Event: "add", range: 1, info: {"AddReplica":{"node_id":2,"store_id":2,"replica_id":2},"UpdatedDesc":{"range_id":1,"end_key":"kw==","replicas":[{"node_id":1,"store_id":1,"replica_id":1},{"node_id":2,"store_id":2,"replica_id":2}],"next_replica_id":3}} | |
| I160906 11:36:34.636610 1179 gossip/server.go:289 [n1] node 1: received [store:3] from node 3 (1 fresh) | |
| I160906 11:36:34.640451 1063 gossip/server.go:162 [n1] node 1: returning 2 info(s) to node 2: [store:3 node:3] | |
| I160906 11:36:34.643800 1074 gossip/client.go:220 [n2] node 2: received [store:3 node:3] from node 1 (1 fresh) | |
| I160906 11:36:34.684735 984 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:34.509681Z/173969882013138944/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:34.509681Z/173969882013138944/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:34.509681Z/173969882013138944/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:34.509681Z/173969882013138944/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:34.509681Z/173969882013138944/6/1,/Min), pErr=<nil> | |
| I160906 11:36:34.702521 1022 storage/replica.go:1600 [n1,s1,r1{/Min-/Table/11}] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:2} for range 1: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}] | |
| I160906 11:36:34.706968 983 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: EndTransaction [/Local/Range/""/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:34.710365 1289 storage/replica.go:2867 gossiping sentinel from store 1, range 1 | |
| I160906 11:36:34.711389 1289 storage/replica.go:2874 gossiping first range from store 1, range 1: [{1 1 1} {2 2 2}] | |
| I160906 11:36:34.712206 1292 kv/dist_sender.go:223 [n1] gossipped first range descriptor: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}] | |
| I160906 11:36:34.713188 1063 gossip/server.go:162 [n1] node 1: returning 4 info(s) to node 2: [first-range node:3 store:3 sentinel] | |
| I160906 11:36:34.730766 1233 rpc/clock_offset.go:147 0 of 0 nodes are within the maximum offset of 50ms | |
| I160906 11:36:34.731798 1074 gossip/client.go:220 [n2] node 2: received [node:3 sentinel store:3 first-range] from node 1 (2 fresh) | |
| I160906 11:36:34.732066 1261 kv/dist_sender.go:223 [n2] gossipped first range descriptor: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}] | |
| I160906 11:36:34.734861 1290 kv/dist_sender.go:1036 application error: end key /Local/Range/""/"rdsc\x00" must be greater than start /Local/Range/""/RangeDescriptor | |
| I160906 11:36:34.735070 1290 kv/dist_sender.go:771 reply error GC [/Local/Range/""/RangeDescriptor,/Local/Range/""/"rdsc\x00"): end key /Local/Range/""/"rdsc\x00" must be greater than start /Local/Range/""/RangeDescriptor | |
| I160906 11:36:34.735199 1290 internal/client/db.go:498 failed batch: end key /Local/Range/""/"rdsc\x00" must be greater than start /Local/Range/""/RangeDescriptor | |
| W160906 11:36:34.735253 1290 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/""/"rdsc\x00" must be greater than start /Local/Range/""/RangeDescriptor | |
| I160906 11:36:34.735478 1022 storage/replicate_queue.go:154 store=1:1 2{/Table/11-/Table/12}: adding replica to {NodeID:2 StoreID:2 ReplicaID:0} due to under-replication | |
| I160906 11:36:34.736529 1177 gossip/server.go:162 [n1] node 1: returning 2 info(s) to node 3: [first-range sentinel] | |
| I160906 11:36:34.737835 1293 storage/reservation.go:197 new reservation added: header:<node_id:2 store_id:2 > from_node_id:1 from_store_id:1 range_id:2 | |
| I160906 11:36:34.739797 1294 storage/replica_raftstorage.go:488 generated snapshot e4793d94-7a5f-4573-acf1-770574128aab for range 2 at index 15 in 218.161µs. 0 log entries | |
| I160906 11:36:34.756942 1297 storage/replica_raftstorage.go:618 store=2:2 2{/Min-/Min}: with replicaID [?], applying preemptive snapshot at index 15 (id=e4793d94, encoded size=16, 1 rocksdb batches, 5 log entries) | |
| I160906 11:36:34.758835 1297 storage/replica_raftstorage.go:621 store=2:2 2{/Table/11-/Table/12}: with replicaID [?], applied preemptive snapshot in 0.002s | |
| I160906 11:36:34.759654 1176 gossip/client.go:220 [n3] node 3: received [first-range sentinel] from node 1 (2 fresh) | |
| I160906 11:36:34.760004 1262 kv/dist_sender.go:223 [n3] gossipped first range descriptor: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}] | |
| I160906 11:36:34.762374 1022 storage/store.go:2634 streamed snapshot: kv pairs: 10, log entries: 5 | |
| I160906 11:36:34.771038 1022 storage/replica_command.go:3058 change replicas of 2: read existing descriptor range_id:2 start_key:"\223" end_key:"\224" replicas:<node_id:1 store_id:1 replica_id:1 > next_replica_id:2 | |
| I160906 11:36:34.776452 1305 storage/raft_transport.go:405 raft transport stream to node 1 established | |
| I160906 11:36:34.816121 1125 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: BeginTransaction [/Local/Range/""/RangeDescriptor,/Min), ConditionalPut [/Local/Range/""/RangeDescriptor,/Min), Put [/Meta2/Table/11,/Min), Put [/Meta1/Max,/Min), pErr=<nil> | |
| I160906 11:36:34.818354 1125 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: EndTransaction [/Local/Range/""/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:34.819758 985 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/12,/Min), pErr=<nil> | |
| I160906 11:36:34.819929 1265 storage/replica_trigger.go:458 unable to gossip first range; hasLease=false, err=<nil> | |
| I160906 11:36:34.823045 984 storage/replica.go:2331 [n1,s1,r2{/Table/11-/Table/12}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x93"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:34.824062 1022 storage/log.go:85 Range Event: "add", range: 2, info: {"AddReplica":{"node_id":2,"store_id":2,"replica_id":2},"UpdatedDesc":{"range_id":2,"start_key":"kw==","end_key":"lA==","replicas":[{"node_id":1,"store_id":1,"replica_id":1},{"node_id":2,"store_id":2,"replica_id":2}],"next_replica_id":3}} | |
| I160906 11:36:34.833355 983 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:34.769363Z/173969882579369984/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:34.769363Z/173969882579369984/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:34.769363Z/173969882579369984/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:34.769363Z/173969882579369984/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:34.769363Z/173969882579369984/6/1,/Min), pErr=<nil> | |
| I160906 11:36:34.855859 1127 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/12,/Min), pErr=<nil> | |
| I160906 11:36:34.858844 1022 storage/replica.go:1600 [n1,s1,r2{/Table/11-/Table/12}] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:2} for range 2: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}] | |
| I160906 11:36:34.915449 985 storage/replica.go:2331 [n1,s1,r2{/Table/11-/Table/12}] applied part of replica change txn: EndTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:34.929315 1022 storage/replicate_queue.go:154 store=1:1 3{/Table/12-/Table/13}: adding replica to {NodeID:3 StoreID:3 ReplicaID:0} due to under-replication | |
| I160906 11:36:34.929474 1022 rpc/context.go:203 dialing 127.0.0.1:59236 | |
| I160906 11:36:35.139122 1325 storage/reservation.go:197 new reservation added: header:<node_id:3 store_id:3 > from_node_id:1 from_store_id:1 range_id:3 range_size:1295 | |
| I160906 11:36:35.141936 1317 rpc/clock_offset.go:147 0 of 0 nodes are within the maximum offset of 50ms | |
| I160906 11:36:35.143389 1326 storage/replica_raftstorage.go:488 generated snapshot 188eaf66-bd23-4d70-9091-3df11b75d10b for range 3 at index 19 in 337.807µs. 0 log entries | |
| I160906 11:36:35.169281 1313 kv/dist_sender.go:1036 application error: end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| I160906 11:36:35.169625 1313 kv/dist_sender.go:771 reply error GC [/Local/Range/"\x93"/RangeDescriptor,/Local/Range/"\x93"/"rdsc\x00"): end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| I160906 11:36:35.169792 1313 internal/client/db.go:498 failed batch: end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| W160906 11:36:35.169890 1313 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| I160906 11:36:35.172196 1022 storage/store.go:2634 streamed snapshot: kv pairs: 25, log entries: 9 | |
| I160906 11:36:35.180062 1331 storage/replica_raftstorage.go:618 store=3:3 3{/Min-/Min}: with replicaID [?], applying preemptive snapshot at index 19 (id=188eaf66, encoded size=16, 1 rocksdb batches, 9 log entries) | |
| I160906 11:36:35.189349 1331 storage/replica_raftstorage.go:621 store=3:3 3{/Table/12-/Table/13}: with replicaID [?], applied preemptive snapshot in 0.009s | |
| I160906 11:36:35.192674 1022 storage/replica_command.go:3058 change replicas of 3: read existing descriptor range_id:3 start_key:"\224" end_key:"\225" replicas:<node_id:1 store_id:1 replica_id:1 > next_replica_id:2 | |
| I160906 11:36:35.197031 984 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/13,/Min), pErr=<nil> | |
| I160906 11:36:35.206655 985 storage/replica.go:2331 [n1,s1,r3{/Table/12-/Table/13}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x94"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x94"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:35.210547 1022 storage/log.go:85 Range Event: "add", range: 3, info: {"AddReplica":{"node_id":3,"store_id":3,"replica_id":2},"UpdatedDesc":{"range_id":3,"start_key":"lA==","end_key":"lQ==","replicas":[{"node_id":1,"store_id":1,"replica_id":1},{"node_id":3,"store_id":3,"replica_id":2}],"next_replica_id":3}} | |
| I160906 11:36:35.241967 1124 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/13,/Min), pErr=<nil> | |
| I160906 11:36:35.256765 983 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:35.190907Z/173969883930722304/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.190907Z/173969883930722304/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.190907Z/173969883930722304/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.190907Z/173969883930722304/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.190907Z/173969883930722304/6/1,/Min), pErr=<nil> | |
| I160906 11:36:35.260604 1022 storage/replica.go:1600 [n1,s1,r3{/Table/12-/Table/13}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:2} for range 3: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:2}] | |
| I160906 11:36:35.262231 982 storage/replica.go:2331 [n1,s1,r3{/Table/12-/Table/13}] applied part of replica change txn: EndTransaction [/Local/Range/"\x94"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:35.269821 1022 storage/replicate_queue.go:154 store=1:1 4{/Table/13-/Table/14}: adding replica to {NodeID:3 StoreID:3 ReplicaID:0} due to under-replication | |
| I160906 11:36:35.288362 1333 storage/reservation.go:197 new reservation added: header:<node_id:3 store_id:3 > from_node_id:1 from_store_id:1 range_id:4 range_size:3853 | |
| I160906 11:36:35.290826 1334 storage/replica_raftstorage.go:488 generated snapshot c5dd3b0d-0ad2-4a2d-913a-72ae45518d7f for range 4 at index 20 in 504.593µs. 0 log entries | |
| I160906 11:36:35.306247 1332 kv/dist_sender.go:1036 application error: end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| I160906 11:36:35.310771 1332 kv/dist_sender.go:771 reply error GC [/Local/Range/"\x94"/RangeDescriptor,/Local/Range/"\x94"/"rdsc\x00"): end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| I160906 11:36:35.311339 1332 internal/client/db.go:498 failed batch: end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| W160906 11:36:35.327184 1332 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| I160906 11:36:35.334473 1022 storage/store.go:2634 streamed snapshot: kv pairs: 54, log entries: 10 | |
| I160906 11:36:35.336055 1346 storage/replica_raftstorage.go:618 store=3:3 4{/Min-/Min}: with replicaID [?], applying preemptive snapshot at index 20 (id=c5dd3b0d, encoded size=16, 1 rocksdb batches, 10 log entries) | |
| I160906 11:36:35.337792 1346 storage/replica_raftstorage.go:621 store=3:3 4{/Table/13-/Table/14}: with replicaID [?], applied preemptive snapshot in 0.002s | |
| I160906 11:36:35.341117 1022 storage/replica_command.go:3058 change replicas of 4: read existing descriptor range_id:4 start_key:"\225" end_key:"\226" replicas:<node_id:1 store_id:1 replica_id:1 > next_replica_id:2 | |
| I160906 11:36:35.345607 985 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/14,/Min), pErr=<nil> | |
| I160906 11:36:35.350566 983 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x95"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x95"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:35.358531 1022 storage/log.go:85 Range Event: "add", range: 4, info: {"AddReplica":{"node_id":3,"store_id":3,"replica_id":2},"UpdatedDesc":{"range_id":4,"start_key":"lQ==","end_key":"lg==","replicas":[{"node_id":1,"store_id":1,"replica_id":1},{"node_id":3,"store_id":3,"replica_id":2}],"next_replica_id":3}} | |
| I160906 11:36:35.382661 982 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:35.339402Z/173969884377939968/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.339402Z/173969884377939968/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.339402Z/173969884377939968/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.339402Z/173969884377939968/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.339402Z/173969884377939968/6/1,/Min), pErr=<nil> | |
| I160906 11:36:35.383867 1125 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/14,/Min), pErr=<nil> | |
| I160906 11:36:35.387436 1022 storage/replica.go:1600 [n1,s1,r4{/Table/13-/Table/14}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:2} for range 4: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:2}] | |
| I160906 11:36:35.399553 984 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: EndTransaction [/Local/Range/"\x95"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:35.402594 1022 storage/replicate_queue.go:154 store=1:1 5{/Table/14-/Max}: adding replica to {NodeID:2 StoreID:2 ReplicaID:0} due to under-replication | |
| I160906 11:36:35.406213 1338 storage/reservation.go:197 new reservation added: header:<node_id:2 store_id:2 > from_node_id:1 from_store_id:1 range_id:5 | |
| I160906 11:36:35.408016 1339 storage/replica_raftstorage.go:488 generated snapshot 31956b76-60a5-436b-ac65-2e370d62e1a4 for range 5 at index 11 in 241.771µs. 0 log entries | |
| I160906 11:36:35.411104 1022 storage/store.go:2634 streamed snapshot: kv pairs: 8, log entries: 1 | |
| I160906 11:36:35.412542 1342 storage/replica_raftstorage.go:618 store=2:2 5{/Min-/Min}: with replicaID [?], applying preemptive snapshot at index 11 (id=31956b76, encoded size=16, 1 rocksdb batches, 1 log entries) | |
| I160906 11:36:35.413848 1342 storage/replica_raftstorage.go:621 store=2:2 5{/Table/14-/Max}: with replicaID [?], applied preemptive snapshot in 0.001s | |
| I160906 11:36:35.428742 1022 kv/dist_sender.go:1036 application error: key range /Local/Range/"\x96"/RangeDescriptor-/Min outside of bounds of range /Table/13-/Table/14 | |
| I160906 11:36:35.428961 1022 kv/dist_sender.go:771 reply error Get [/Local/Range/"\x96"/RangeDescriptor,/Min): key range /Local/Range/"\x96"/RangeDescriptor-/Min outside of bounds of range /Table/13-/Table/14 | |
| W160906 11:36:35.429382 1022 kv/dist_sender.go:816 key range /Local/Range/"\x96"/RangeDescriptor-/Min outside of bounds of range /Table/13-/Table/14 | |
| I160906 11:36:35.430975 1022 storage/replica_command.go:3058 change replicas of 5: read existing descriptor range_id:5 start_key:"\226" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > next_replica_id:2 | |
| I160906 11:36:35.437555 1337 kv/dist_sender.go:1036 application error: end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| I160906 11:36:35.438612 1337 kv/dist_sender.go:771 reply error GC [/Local/Range/"\x95"/RangeDescriptor,/Local/Range/"\x95"/"rdsc\x00"): end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| I160906 11:36:35.444299 1337 internal/client/db.go:498 failed batch: end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| W160906 11:36:35.444384 1337 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| I160906 11:36:35.448071 982 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Max,/Min), pErr=<nil> | |
| I160906 11:36:35.451488 984 storage/replica.go:2331 [n1,s1,r5{/Table/14-/Max}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x96"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x96"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:35.459302 1022 storage/log.go:85 Range Event: "add", range: 5, info: {"AddReplica":{"node_id":2,"store_id":2,"replica_id":2},"UpdatedDesc":{"range_id":5,"start_key":"lg==","end_key":"//8=","replicas":[{"node_id":1,"store_id":1,"replica_id":1},{"node_id":2,"store_id":2,"replica_id":2}],"next_replica_id":3}} | |
| I160906 11:36:35.467864 1125 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Max,/Min), pErr=<nil> | |
| I160906 11:36:35.491195 1365 storage/raft_transport.go:405 raft transport stream to node 1 established | |
| I160906 11:36:35.497042 1244 storage/replica.go:2331 [n3,s3,r4{/Table/13-/Table/14}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x95"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x95"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:35.499271 1244 storage/replica.go:2331 [n3,s3,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:35.339402Z/173969884377939968/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.339402Z/173969884377939968/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.339402Z/173969884377939968/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.339402Z/173969884377939968/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.339402Z/173969884377939968/6/1,/Min), pErr=<nil> | |
| I160906 11:36:35.501878 1244 storage/replica.go:2331 [n3,s3,r4{/Table/13-/Table/14}] applied part of replica change txn: EndTransaction [/Local/Range/"\x95"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:35.507715 984 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:35.416512Z/173969884672589824/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.416512Z/173969884672589824/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.416512Z/173969884672589824/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.416512Z/173969884672589824/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.416512Z/173969884672589824/6/1,/Min), pErr=<nil> | |
| I160906 11:36:35.513948 1022 storage/replica.go:1600 [n1,s1,r5{/Table/14-/Max}] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:2} for range 5: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}] | |
| I160906 11:36:35.520495 985 storage/replica.go:2331 [n1,s1,r5{/Table/14-/Max}] applied part of replica change txn: EndTransaction [/Local/Range/"\x96"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:35.529894 1022 storage/queue.go:563 [replicate] purgatory is now empty | |
| I160906 11:36:35.530738 1246 storage/replica.go:2331 [n3,s3,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:35.416512Z/173969884672589824/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.416512Z/173969884672589824/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.416512Z/173969884672589824/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.416512Z/173969884672589824/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.416512Z/173969884672589824/6/1,/Min), pErr=<nil> | |
| I160906 11:36:35.532209 992 storage/replicate_queue.go:154 store=1:1 1{/Min-/Table/11}: adding replica to {NodeID:3 StoreID:3 ReplicaID:0} due to under-replication | |
| I160906 11:36:35.535947 1371 storage/reservation.go:197 new reservation added: header:<node_id:3 store_id:3 > from_node_id:1 from_store_id:1 range_id:1 range_size:60240 | |
| I160906 11:36:35.540656 1372 storage/replica_raftstorage.go:488 generated snapshot e464e21c-2558-4c48-baa0-076a492e1a3d for range 1 at index 57 in 238.131µs. 0 log entries | |
| I160906 11:36:35.552538 1370 kv/dist_sender.go:1036 application error: end key /Local/Range/"\x96"/"rdsc\x00" must be greater than start /Local/Range/"\x96"/RangeDescriptor | |
| I160906 11:36:35.552844 1370 kv/dist_sender.go:771 reply error GC [/Local/Range/"\x96"/RangeDescriptor,/Local/Range/"\x96"/"rdsc\x00"): end key /Local/Range/"\x96"/"rdsc\x00" must be greater than start /Local/Range/"\x96"/RangeDescriptor | |
| I160906 11:36:35.553032 1370 internal/client/db.go:498 failed batch: end key /Local/Range/"\x96"/"rdsc\x00" must be greater than start /Local/Range/"\x96"/RangeDescriptor | |
| W160906 11:36:35.553135 1370 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/"\x96"/"rdsc\x00" must be greater than start /Local/Range/"\x96"/RangeDescriptor | |
| I160906 11:36:35.580156 992 storage/store.go:2634 streamed snapshot: kv pairs: 584, log entries: 47 | |
| I160906 11:36:35.582962 1375 storage/replica_raftstorage.go:618 store=3:3 1{/Min-/Min}: with replicaID [?], applying preemptive snapshot at index 57 (id=e464e21c, encoded size=16, 1 rocksdb batches, 47 log entries) | |
| I160906 11:36:35.601357 1375 storage/replica_raftstorage.go:621 store=3:3 1{/Min-/Table/11}: with replicaID [?], applied preemptive snapshot in 0.005s | |
| I160906 11:36:35.604941 992 storage/replica_command.go:3058 change replicas of 1: read existing descriptor range_id:1 start_key:"" end_key:"\223" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:3 | |
| I160906 11:36:35.614250 982 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: BeginTransaction [/Local/Range/""/RangeDescriptor,/Min), ConditionalPut [/Local/Range/""/RangeDescriptor,/Min), Put [/Meta2/Table/11,/Min), Put [/Meta1/Max,/Min), pErr=<nil> | |
| I160906 11:36:35.618173 992 storage/log.go:85 Range Event: "add", range: 1, info: {"AddReplica":{"node_id":3,"store_id":3,"replica_id":3},"UpdatedDesc":{"range_id":1,"end_key":"kw==","replicas":[{"node_id":1,"store_id":1,"replica_id":1},{"node_id":2,"store_id":2,"replica_id":2},{"node_id":3,"store_id":3,"replica_id":3}],"next_replica_id":4}} | |
| I160906 11:36:35.622421 1125 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: BeginTransaction [/Local/Range/""/RangeDescriptor,/Min), ConditionalPut [/Local/Range/""/RangeDescriptor,/Min), Put [/Meta2/Table/11,/Min), Put [/Meta1/Max,/Min), pErr=<nil> | |
| I160906 11:36:35.658947 983 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:35.603221Z/173969885262479360/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.603221Z/173969885262479360/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.603221Z/173969885262479360/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.603221Z/173969885262479360/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.603221Z/173969885262479360/6/1,/Min), pErr=<nil> | |
| I160906 11:36:35.661720 992 storage/replica.go:1600 [n1,s1,r1{/Min-/Table/11}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:3} for range 1: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:3}] | |
| I160906 11:36:35.669528 1246 storage/replica.go:2331 [n3,s3,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:35.603221Z/173969885262479360/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.603221Z/173969885262479360/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.603221Z/173969885262479360/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.603221Z/173969885262479360/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:35.603221Z/173969885262479360/6/1,/Min), pErr=<nil> | |
| I160906 11:36:35.680108 984 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: EndTransaction [/Local/Range/""/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:35.685409 1377 storage/replica.go:2867 gossiping sentinel from store 1, range 1 | |
| I160906 11:36:35.685858 1377 storage/replica.go:2874 gossiping first range from store 1, range 1: [{1 1 1} {2 2 2} {3 3 3}] | |
| I160906 11:36:35.686349 1380 kv/dist_sender.go:223 [n1] gossipped first range descriptor: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:3}] | |
| I160906 11:36:35.686657 1063 gossip/server.go:162 [n1] node 1: returning 4 info(s) to node 2: [store:3 first-range sentinel node:3] | |
| I160906 11:36:35.713491 992 storage/replicate_queue.go:154 store=1:1 2{/Table/11-/Table/12}: adding replica to {NodeID:3 StoreID:3 ReplicaID:0} due to under-replication | |
| I160906 11:36:35.716172 1177 gossip/server.go:162 [n1] node 1: returning 2 info(s) to node 3: [first-range sentinel] | |
| I160906 11:36:35.717949 1124 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: EndTransaction [/Local/Range/""/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:35.720833 1381 storage/replica_trigger.go:458 unable to gossip first range; hasLease=false, err=<nil> | |
| I160906 11:36:35.722831 1176 gossip/client.go:220 [n3] node 3: received [sentinel first-range] from node 1 (2 fresh) | |
| I160906 11:36:35.723194 1383 kv/dist_sender.go:223 [n3] gossipped first range descriptor: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:3}] | |
| I160906 11:36:35.726553 1384 storage/reservation.go:197 new reservation added: header:<node_id:3 store_id:3 > from_node_id:1 from_store_id:1 range_id:2 | |
| I160906 11:36:35.732110 1074 gossip/client.go:220 [n2] node 2: received [first-range node:3 sentinel store:3] from node 1 (2 fresh) | |
| I160906 11:36:35.732228 1348 kv/dist_sender.go:223 [n2] gossipped first range descriptor: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:3}] | |
| I160906 11:36:35.734101 1349 storage/replica_raftstorage.go:488 generated snapshot d913f671-c9e5-4486-ad18-ccd8165f0aff for range 2 at index 17 in 291.506µs. 0 log entries | |
| I160906 11:36:35.741899 992 storage/store.go:2634 streamed snapshot: kv pairs: 12, log entries: 7 | |
| I160906 11:36:35.743450 1352 storage/replica_raftstorage.go:618 store=3:3 2{/Min-/Min}: with replicaID [?], applying preemptive snapshot at index 17 (id=d913f671, encoded size=16, 1 rocksdb batches, 7 log entries) | |
| I160906 11:36:35.745565 1352 storage/replica_raftstorage.go:621 store=3:3 2{/Table/11-/Table/12}: with replicaID [?], applied preemptive snapshot in 0.002s | |
| I160906 11:36:35.749635 992 storage/replica_command.go:3058 change replicas of 2: read existing descriptor range_id:2 start_key:"\223" end_key:"\224" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:3 | |
| I160906 11:36:35.761268 1378 kv/dist_sender.go:1036 application error: end key /Local/Range/""/"rdsc\x00" must be greater than start /Local/Range/""/RangeDescriptor | |
| I160906 11:36:35.763203 1378 kv/dist_sender.go:771 reply error GC [/Local/Range/""/RangeDescriptor,/Local/Range/""/"rdsc\x00"): end key /Local/Range/""/"rdsc\x00" must be greater than start /Local/Range/""/RangeDescriptor | |
| I160906 11:36:35.763528 1378 internal/client/db.go:498 failed batch: end key /Local/Range/""/"rdsc\x00" must be greater than start /Local/Range/""/RangeDescriptor | |
| W160906 11:36:35.763623 1378 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/""/"rdsc\x00" must be greater than start /Local/Range/""/RangeDescriptor | |
| I160906 11:36:35.774951 982 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/12,/Min), pErr=<nil> | |
| I160906 11:36:35.786437 1247 storage/replica.go:2331 [n3,s3,r1{/Min-/Table/11}] applied part of replica change txn: BeginTransaction [/Local/Range/""/RangeDescriptor,/Min), ConditionalPut [/Local/Range/""/RangeDescriptor,/Min), Put [/Meta2/Table/11,/Min), Put [/Meta1/Max,/Min), pErr=<nil> | |
| I160906 11:36:35.796333 1247 storage/replica.go:2331 [n3,s3,r1{/Min-/Table/11}] applied part of replica change txn: EndTransaction [/Local/Range/""/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:35.797833 1247 storage/replica.go:2331 [n3,s3,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/12,/Min), pErr=<nil> | |
| I160906 11:36:35.798480 1385 storage/replica_trigger.go:458 unable to gossip first range; hasLease=false, err=<nil> | |
| I160906 11:36:35.801863 1126 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/12,/Min), pErr=<nil> | |
| W160906 11:36:35.802610 971 raft/raft.go:647 store=1:1 2{/Table/11-/Table/12}: 1 stepped down to follower since quorum is not active | |
| I160906 11:36:35.803357 983 storage/replica.go:1950 [n1,s1,r2{/Table/11-/Table/12}] pending commands: refurbished 0, reproposing 1 (at 17.6); reasonNewLeader | |
| I160906 11:36:35.807713 1001 rpc/clock_offset.go:147 1 of 1 nodes are within the maximum offset of 50ms | |
| W160906 11:36:35.987653 971 raft/raft.go:647 store=1:1 3{/Table/12-/Table/13}: 1 stepped down to follower since quorum is not active | |
| W160906 11:36:36.187636 971 raft/raft.go:647 store=1:1 5{/Table/14-/Max}: 1 stepped down to follower since quorum is not active | |
| I160906 11:36:36.568814 1058 rpc/clock_offset.go:147 1 of 1 nodes are within the maximum offset of 50ms | |
| I160906 11:36:36.929250 1104 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:37.271558 1148 rpc/clock_offset.go:147 1 of 1 nodes are within the maximum offset of 50ms | |
| I160906 11:36:37.285845 1188 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:37.545760 947 kv/txn_coord_sender.go:251 [n1] txn coordinator: 4.18 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 130ms/48ms/223ms avg/σ/max duration, 0.1/0.3/1 avg/σ/max restarts (11 samples) | |
| I160906 11:36:37.733300 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:38.144529 1317 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:38.463626 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.77 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 74ms/0s/74ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:36:38.778272 992 kv/dist_sender.go:1036 application error: context deadline exceeded | |
| I160906 11:36:38.778495 992 kv/dist_sender.go:771 reply error BeginTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x93"/RangeDescriptor,/Min), Put [/Meta2/Table/12,/Min): context deadline exceeded | |
| I160906 11:36:38.778661 992 internal/client/db.go:498 failed batch: context deadline exceeded | |
| E160906 11:36:38.779127 992 storage/queue.go:506 [replicate] on store=1:1 2{/Table/11-/Table/12}: change replicas of range 2 failed: context deadline exceeded | |
| I160906 11:36:38.779919 992 storage/replicate_queue.go:154 store=1:1 5{/Table/14-/Max}: adding replica to {NodeID:3 StoreID:3 ReplicaID:0} due to under-replication | |
| I160906 11:36:38.781407 1394 storage/reservation.go:197 new reservation added: header:<node_id:3 store_id:3 > from_node_id:1 from_store_id:1 range_id:5 | |
| I160906 11:36:38.783086 1395 storage/replica_raftstorage.go:488 generated snapshot 74c79423-9311-4562-ac09-7a7f91218fb6 for range 5 at index 13 in 236.303µs. 0 log entries | |
| I160906 11:36:38.786120 992 storage/store.go:2634 streamed snapshot: kv pairs: 10, log entries: 3 | |
| I160906 11:36:38.787809 1398 storage/replica_raftstorage.go:618 store=3:3 5{/Min-/Min}: with replicaID [?], applying preemptive snapshot at index 13 (id=74c79423, encoded size=16, 1 rocksdb batches, 3 log entries) | |
| I160906 11:36:38.789605 1398 storage/replica_raftstorage.go:621 store=3:3 5{/Table/14-/Max}: with replicaID [?], applied preemptive snapshot in 0.002s | |
| I160906 11:36:38.793155 992 storage/replica_command.go:3058 change replicas of 5: read existing descriptor range_id:5 start_key:"\226" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:3 | |
| I160906 11:36:38.799826 985 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Max,/Min), pErr=<nil> | |
| I160906 11:36:38.803171 1244 storage/replica.go:2331 [n3,s3,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Max,/Min), pErr=<nil> | |
| I160906 11:36:38.809417 1125 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Max,/Min), pErr=<nil> | |
| I160906 11:36:38.813258 1001 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:38.883981 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.77 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 37ms/0s/37ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:36:39.571376 1058 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:39.931558 1104 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:40.274183 1148 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:40.288326 1188 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:40.735748 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:41.147006 1317 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:41.595865 1126 storage/replica.go:2331 [n2,s2,r2{/Table/11-/Table/12}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x93"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:41.596929 1126 storage/replica.go:2331 [n2,s2,r2{/Table/11-/Table/12}] applied part of replica change txn: EndTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:41.599411 985 storage/replica.go:2331 [n1,s1,r2{/Table/11-/Table/12}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x93"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:41.599903 985 storage/replica.go:2237 [n1,s1,r2{/Table/11-/Table/12}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:41.600753 985 storage/replica.go:2272 [n1,s1,r2{/Table/11-/Table/12}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:41.602421 1124 storage/replica.go:2331 [n2,s2,r2{/Table/11-/Table/12}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x93"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:41.607548 1124 storage/replica.go:2237 [n2,s2,r2{/Table/11-/Table/12}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:41.608475 1124 storage/replica.go:2272 [n2,s2,r2{/Table/11-/Table/12}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:41.793557 1244 storage/replica.go:2331 [n3,s3,r3{/Table/12-/Table/13}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x94"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x94"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:41.794493 1244 storage/replica.go:2331 [n3,s3,r3{/Table/12-/Table/13}] applied part of replica change txn: EndTransaction [/Local/Range/"\x94"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:41.796346 984 storage/replica.go:2237 [n1,s1,r3{/Table/12-/Table/13}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:41.797194 984 storage/replica.go:2272 [n1,s1,r3{/Table/12-/Table/13}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:41.798193 1246 storage/replica.go:2237 [n3,s3,r3{/Table/12-/Table/13}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:41.799052 1246 storage/replica.go:2272 [n3,s3,r3{/Table/12-/Table/13}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:41.802147 992 kv/dist_sender.go:1036 application error: context deadline exceeded | |
| I160906 11:36:41.802420 992 kv/dist_sender.go:771 reply error BeginTransaction [/Local/Range/"\x96"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x96"/RangeDescriptor,/Min), Put [/Meta2/Max,/Min): context deadline exceeded | |
| I160906 11:36:41.802672 992 internal/client/db.go:498 failed batch: context deadline exceeded | |
| E160906 11:36:41.803113 992 storage/queue.go:506 [replicate] on store=1:1 5{/Table/14-/Max}: change replicas of range 5 failed: context deadline exceeded | |
| I160906 11:36:41.808374 992 storage/replicate_queue.go:154 store=1:1 4{/Table/13-/Table/14}: adding replica to {NodeID:2 StoreID:2 ReplicaID:0} due to under-replication | |
| I160906 11:36:41.810085 1389 storage/reservation.go:197 new reservation added: header:<node_id:2 store_id:2 > from_node_id:1 from_store_id:1 range_id:4 range_size:4841 | |
| I160906 11:36:41.813955 1390 storage/replica_raftstorage.go:488 generated snapshot 38cb1a85-855d-4214-9865-943a2f99254f for range 4 at index 29 in 213.482µs. 0 log entries | |
| I160906 11:36:41.818283 992 storage/store.go:2634 streamed snapshot: kv pairs: 66, log entries: 19 | |
| I160906 11:36:41.820110 1393 storage/replica_raftstorage.go:618 store=2:2 4{/Min-/Min}: with replicaID [?], applying preemptive snapshot at index 29 (id=38cb1a85, encoded size=16, 1 rocksdb batches, 19 log entries) | |
| I160906 11:36:41.822273 1393 storage/replica_raftstorage.go:621 store=2:2 4{/Table/13-/Table/14}: with replicaID [?], applied preemptive snapshot in 0.002s | |
| I160906 11:36:41.834766 992 storage/replica_command.go:3058 change replicas of 4: read existing descriptor range_id:4 start_key:"\225" end_key:"\226" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:2 > next_replica_id:3 | |
| I160906 11:36:41.838975 1001 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:41.853648 984 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/14,/Min), pErr=<nil> | |
| I160906 11:36:41.868075 1124 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/14,/Min), pErr=<nil> | |
| I160906 11:36:41.870974 1247 storage/replica.go:2331 [n3,s3,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/14,/Min), pErr=<nil> | |
| I160906 11:36:41.875577 984 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x95"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x95"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:41.879035 1244 storage/replica.go:2331 [n3,s3,r4{/Table/13-/Table/14}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x95"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x95"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:41.897162 992 storage/log.go:85 Range Event: "add", range: 4, info: {"AddReplica":{"node_id":2,"store_id":2,"replica_id":3},"UpdatedDesc":{"range_id":4,"start_key":"lQ==","end_key":"lg==","replicas":[{"node_id":1,"store_id":1,"replica_id":1},{"node_id":3,"store_id":3,"replica_id":2},{"node_id":2,"store_id":2,"replica_id":3}],"next_replica_id":4}} | |
| I160906 11:36:41.909853 985 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/6/1,/Min), pErr=<nil> | |
| I160906 11:36:41.930435 992 storage/replica.go:1600 [n1,s1,r4{/Table/13-/Table/14}] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:3} for range 4: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:2} {NodeID:2 StoreID:2 ReplicaID:3}] | |
| I160906 11:36:41.933733 1246 storage/replica.go:2331 [n3,s3,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/6/1,/Min), pErr=<nil> | |
| I160906 11:36:41.936300 983 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: EndTransaction [/Local/Range/"\x95"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:41.969944 1244 storage/replica.go:2331 [n3,s3,r4{/Table/13-/Table/14}] applied part of replica change txn: EndTransaction [/Local/Range/"\x95"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:42.027795 1407 kv/dist_sender.go:1036 application error: end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| I160906 11:36:42.028144 1407 kv/dist_sender.go:771 reply error GC [/Local/Range/"\x95"/RangeDescriptor,/Local/Range/"\x95"/"rdsc\x00"): end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| I160906 11:36:42.028422 1407 internal/client/db.go:498 failed batch: end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| W160906 11:36:42.028494 1407 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/"\x95"/"rdsc\x00" must be greater than start /Local/Range/"\x95"/RangeDescriptor | |
| I160906 11:36:42.057372 992 storage/replicate_queue.go:154 store=1:1 3{/Table/12-/Table/13}: adding replica to {NodeID:2 StoreID:2 ReplicaID:0} due to under-replication | |
| I160906 11:36:42.078907 1409 storage/reservation.go:197 new reservation added: header:<node_id:2 store_id:2 > from_node_id:1 from_store_id:1 range_id:3 range_size:1295 | |
| I160906 11:36:42.099876 1412 storage/replica_raftstorage.go:488 generated snapshot badc94e8-55e4-4508-b71c-03ec08a1b4f2 for range 3 at index 23 in 260.944µs. 0 log entries | |
| I160906 11:36:42.107412 1125 storage/replica.go:2331 [n2,s2,r5{/Table/14-/Max}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x96"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x96"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:42.108370 1125 storage/replica.go:2331 [n2,s2,r5{/Table/14-/Max}] applied part of replica change txn: EndTransaction [/Local/Range/"\x96"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:42.110819 992 storage/store.go:2634 streamed snapshot: kv pairs: 27, log entries: 13 | |
| I160906 11:36:42.112395 1415 storage/replica_raftstorage.go:618 store=2:2 3{/Min-/Min}: with replicaID [?], applying preemptive snapshot at index 23 (id=badc94e8, encoded size=16, 1 rocksdb batches, 13 log entries) | |
| I160906 11:36:42.114192 1415 storage/replica_raftstorage.go:621 store=2:2 3{/Table/12-/Table/13}: with replicaID [?], applied preemptive snapshot in 0.002s | |
| I160906 11:36:42.115910 985 storage/replica.go:2237 [n1,s1,r5{/Table/14-/Max}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:42.116768 985 storage/replica.go:2272 [n1,s1,r5{/Table/14-/Max}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:42.117841 1124 storage/replica.go:2237 [n2,s2,r5{/Table/14-/Max}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:42.118660 1124 storage/replica.go:2272 [n2,s2,r5{/Table/14-/Max}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:42.121308 992 storage/replica_command.go:3058 change replicas of 3: read existing descriptor range_id:3 start_key:"\224" end_key:"\225" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:2 > next_replica_id:3 | |
| I160906 11:36:42.127232 982 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/13,/Min), pErr=<nil> | |
| I160906 11:36:42.136109 1127 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/13,/Min), pErr=<nil> | |
| I160906 11:36:42.137882 1246 storage/replica.go:2331 [n3,s3,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/13,/Min), pErr=<nil> | |
| I160906 11:36:42.142847 983 storage/replica.go:2331 [n1,s1,r3{/Table/12-/Table/13}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x94"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x94"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:42.146388 992 storage/log.go:85 Range Event: "add", range: 3, info: {"AddReplica":{"node_id":2,"store_id":2,"replica_id":3},"UpdatedDesc":{"range_id":3,"start_key":"lA==","end_key":"lQ==","replicas":[{"node_id":1,"store_id":1,"replica_id":1},{"node_id":3,"store_id":3,"replica_id":2},{"node_id":2,"store_id":2,"replica_id":3}],"next_replica_id":4}} | |
| I160906 11:36:42.183375 1124 storage/replica.go:2331 [n2,s2,r4{/Table/13-/Table/14}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x95"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x95"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:42.184751 1124 storage/replica.go:2331 [n2,s2,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:41.832976Z/173969905759846400/6/1,/Min), pErr=<nil> | |
| I160906 11:36:42.185731 1124 storage/replica.go:2331 [n2,s2,r4{/Table/13-/Table/14}] applied part of replica change txn: EndTransaction [/Local/Range/"\x95"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:42.188629 983 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/6/1,/Min), pErr=<nil> | |
| I160906 11:36:42.191803 992 storage/replica.go:1600 [n1,s1,r3{/Table/12-/Table/13}] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:3} for range 3: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:2} {NodeID:2 StoreID:2 ReplicaID:3}] | |
| I160906 11:36:42.195055 1126 storage/replica.go:2331 [n2,s2,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/6/1,/Min), pErr=<nil> | |
| I160906 11:36:42.198966 1245 storage/replica.go:2331 [n3,s3,r3{/Table/12-/Table/13}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x94"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x94"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:42.201341 1244 storage/replica.go:2331 [n3,s3,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:42.119494Z/173969906653822976/6/1,/Min), pErr=<nil> | |
| I160906 11:36:42.205427 982 storage/replica.go:2331 [n1,s1,r3{/Table/12-/Table/13}] applied part of replica change txn: EndTransaction [/Local/Range/"\x94"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:42.213189 1246 storage/replica.go:2331 [n3,s3,r3{/Table/12-/Table/13}] applied part of replica change txn: EndTransaction [/Local/Range/"\x94"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:42.228343 1417 kv/dist_sender.go:1036 application error: end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| I160906 11:36:42.228578 1417 kv/dist_sender.go:771 reply error GC [/Local/Range/"\x94"/RangeDescriptor,/Local/Range/"\x94"/"rdsc\x00"): end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| I160906 11:36:42.228727 1417 internal/client/db.go:498 failed batch: end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| W160906 11:36:42.228802 1417 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/"\x94"/"rdsc\x00" must be greater than start /Local/Range/"\x94"/RangeDescriptor | |
| I160906 11:36:42.528299 948 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 2: 0 1 0 5.8 0.0 0.0 5.8 | |
| 3: 0 1 0 5.9 0.0 0.0 5.9 | |
| I160906 11:36:42.571229 947 kv/txn_coord_sender.go:251 [n1] txn coordinator: 3.12 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 129ms/46ms/223ms avg/σ/max duration, 0.1/0.3/1 avg/σ/max restarts (13 samples) | |
| I160906 11:36:42.594433 1058 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:42.598143 1014 server/status/runtime.go:207 runtime stats: 276 MiB RSS, 216 goroutines, 15 MiB/5.8 MiB/32 MiB GO alloc/idle/total, 71 MiB/83 MiB CGO alloc/total, 0.00cgo/sec, 0.00/0.00 %(u/s)time, 0.00 %gc (19x) | |
| I160906 11:36:42.915734 1126 storage/replica.go:2331 [n2,s2,r3{/Table/12-/Table/13}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x94"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x94"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:42.944127 1126 storage/replica.go:2331 [n2,s2,r3{/Table/12-/Table/13}] applied part of replica change txn: EndTransaction [/Local/Range/"\x94"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:42.982466 1104 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:43.320498 1148 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:43.322341 1188 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:43.460744 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 6.8 0.0 0.0 6.8 | |
| I160906 11:36:43.526229 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.55 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 74ms/0s/74ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:36:43.650764 1094 server/status/runtime.go:207 runtime stats: 276 MiB RSS, 216 goroutines, 18 MiB/3.6 MiB/32 MiB GO alloc/idle/total, 70 MiB/83 MiB CGO alloc/total, 0.00cgo/sec, 0.00/0.00 %(u/s)time, 0.00 %gc (19x) | |
| I160906 11:36:43.738243 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:43.905321 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 6.8 0.0 0.0 6.8 | |
| I160906 11:36:43.956497 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.55 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 37ms/0s/37ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:36:44.163601 1095 rpc/context.go:203 dialing 127.0.0.1:59236 | |
| I160906 11:36:44.227568 1095 kv/dist_sender.go:1036 application error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:44.260793 1317 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:44.262735 1095 kv/dist_sender.go:1107 [n2] range 1: caching new lease holder {NodeID:1 StoreID:1 ReplicaID:1} | |
| I160906 11:36:44.350118 982 storage/replica_trigger.go:330 [n1,s1,r2{/Table/11-/Table/12}] range 2: replicaID 1 transfer raft leadership to replicaID 2 | |
| I160906 11:36:44.388175 1223 server/status/runtime.go:207 runtime stats: 287 MiB RSS, 223 goroutines, 24 MiB/1.6 MiB/34 MiB GO alloc/idle/total, 71 MiB/84 MiB CGO alloc/total, 0.00cgo/sec, 0.00/0.00 %(u/s)time, 0.00 %gc (19x) | |
| I160906 11:36:44.418705 1127 storage/replica_trigger.go:308 [n2,s2,r2{/Table/11-/Table/12}] new range lease replica {2 2 2} 2016-09-06 11:36:41.571292356 +0000 UTC 11.741071307s following replica {1 1 1} 1970-01-01 00:00:00 +0000 UTC 409211h36m41.571292356s [physicalTime=2016-09-06 11:36:44.418429348 +0000 UTC] | |
| I160906 11:36:44.478948 1124 storage/replica.go:1950 [n2,s2,r2{/Table/11-/Table/12}] pending commands: refurbished 0, reproposing 1 (at 20.7); reasonNewLeader | |
| I160906 11:36:44.519955 1127 storage/replica.go:1950 [n2,s2,r2{/Table/11-/Table/12}] pending commands: refurbished 0, reproposing 1 (at 20.7); reasonNewLeader | |
| I160906 11:36:44.550624 1429 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:44.613713 1124 storage/replica.go:2237 [n2,s2,r2{/Table/11-/Table/12}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:44.614650 1124 storage/replica.go:2272 [n2,s2,r2{/Table/11-/Table/12}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:44.614845 1124 storage/replica.go:1950 [n2,s2,r2{/Table/11-/Table/12}] pending commands: refurbished 0, reproposing 1 (at 21.7); reasonNewLeaderOrConfigChange | |
| I160906 11:36:44.655820 982 storage/replica.go:2237 [n1,s1,r2{/Table/11-/Table/12}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:44.691557 982 storage/replica.go:2272 [n1,s1,r2{/Table/11-/Table/12}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:44.745926 1127 storage/replica_command.go:1363 [n2,s2,r2{/Table/11-/Table/12}] failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| I160906 11:36:44.748030 1127 storage/replica.go:2237 [n2,s2,r2{/Table/11-/Table/12}] applying command with forced error: storage/replica.go:2193: command observed at lease index 8, but required < 8 | |
| E160906 11:36:44.753304 1127 storage/replica.go:2272 [n2,s2,r2{/Table/11-/Table/12}] error executing raft command: storage/replica.go:2193: command observed at lease index 8, but required < 8 | |
| I160906 11:36:44.783225 1433 kv/dist_sender.go:1036 application error: failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| I160906 11:36:44.792128 1433 kv/dist_sender.go:771 reply error PushTxn [/Local/Range/"\x93"/RangeDescriptor,/Min), PushTxn [/Local/Range/"\x96"/RangeDescriptor,/Min): failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| I160906 11:36:44.800550 1433 internal/client/db.go:498 failed batch: failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| W160906 11:36:44.800859 1433 storage/intent_resolver.go:303 store=2:2 1{/Min-/Table/11}: failed to push during intent resolution: failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| I160906 11:36:44.814579 984 storage/replica_command.go:1363 [n1,s1,r2{/Table/11-/Table/12}] failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| E160906 11:36:44.815370 984 storage/replica.go:2272 [n1,s1,r2{/Table/11-/Table/12}] error executing raft command: failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| I160906 11:36:44.815605 984 storage/replica.go:2237 [n1,s1,r2{/Table/11-/Table/12}] applying command with forced error: storage/replica.go:2193: command observed at lease index 8, but required < 8 | |
| E160906 11:36:44.816490 984 storage/replica.go:2272 [n1,s1,r2{/Table/11-/Table/12}] error executing raft command: storage/replica.go:2193: command observed at lease index 8, but required < 8 | |
| I160906 11:36:44.833075 1224 rpc/context.go:203 dialing 127.0.0.1:41417 | |
| I160906 11:36:44.876624 1453 kv/dist_sender.go:1036 application error: range 2: replica {1 1 1} not lease holder; node_id:2 store_id:2 replica_id:2 is | |
| I160906 11:36:44.876781 1453 kv/dist_sender.go:1107 [n1] range 2: caching new lease holder {NodeID:2 StoreID:2 ReplicaID:2} | |
| I160906 11:36:44.877136 1453 kv/dist_sender.go:771 reply error PushTxn [/Local/Range/"\x93"/RangeDescriptor,/Min), PushTxn [/Local/Range/"\x96"/RangeDescriptor,/Min): failed to send RPC: sending to all 1 replicas failed; last error: range 2: replica {1 1 1} not lease holder; node_id:2 store_id:2 replica_id:2 is | |
| I160906 11:36:44.918622 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:44.923495 1224 kv/dist_sender.go:1107 [n3] range 1: caching new lease holder {NodeID:1 StoreID:1 ReplicaID:1} | |
| I160906 11:36:44.964749 1001 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:45.011261 1457 storage/intent_resolver.go:194 skipping PushTxn for 8f6a0f68-570a-4b3f-b1c0-b518e326e51f; attempt already in flight | |
| I160906 11:36:45.149323 1127 storage/replica_command.go:1363 [n2,s2,r2{/Table/11-/Table/12}] failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| I160906 11:36:45.190653 1453 kv/dist_sender.go:1036 application error: failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| I160906 11:36:45.190986 1453 kv/dist_sender.go:771 reply error PushTxn [/Local/Range/"\x93"/RangeDescriptor,/Min), PushTxn [/Local/Range/"\x96"/RangeDescriptor,/Min): failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| I160906 11:36:45.191252 1453 internal/client/db.go:498 failed batch: failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| W160906 11:36:45.191430 1453 storage/intent_resolver.go:303 store=1:1 1{/Min-/Table/11}: failed to push during intent resolution: failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| I160906 11:36:45.192526 983 storage/replica_command.go:1363 [n1,s1,r2{/Table/11-/Table/12}] failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| E160906 11:36:45.193332 983 storage/replica.go:2272 [n1,s1,r2{/Table/11-/Table/12}] error executing raft command: failed to push "change-replica" id=8f6a0f68 key=/Local/Range/"\x93"/RangeDescriptor rw=true pri=0.01678313 iso=SERIALIZABLE stat=PENDING epo=0 ts=1473161795.747418103,0 orig=1473161795.747418103,0 max=1473161795.747418103,0 wto=false rop=false | |
| I160906 11:36:45.199394 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:45.596875 1058 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:45.984988 1104 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:46.323103 1148 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:46.325280 1188 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:46.740680 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:36:47.263382 1317 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:47.553026 1429 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:47.610813 947 kv/txn_coord_sender.go:251 [n1] txn coordinator: 2.24 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 129ms/46ms/223ms avg/σ/max duration, 0.1/0.3/1 avg/σ/max restarts (13 samples) | |
| I160906 11:36:47.967238 1001 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:48.202067 1438 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:48.542935 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.39 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 74ms/0s/74ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:36:48.599436 1058 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:48.907295 1137 storage/replicate_queue.go:154 store=2:2 2{/Table/11-/Table/12}: adding replica to {NodeID:3 StoreID:3 ReplicaID:0} due to under-replication | |
| I160906 11:36:48.910093 1463 storage/reservation.go:197 new reservation added: header:<node_id:3 store_id:3 > from_node_id:2 from_store_id:2 range_id:2 | |
| I160906 11:36:48.911932 1464 storage/replica_raftstorage.go:488 generated snapshot 5684622a-83c1-48e0-a3cd-af3d81b0b43b for range 2 at index 24 in 255.979µs. 0 log entries | |
| I160906 11:36:48.915716 1137 storage/store.go:2634 streamed snapshot: kv pairs: 15, log entries: 14 | |
| I160906 11:36:48.917087 1467 storage/replica_raftstorage.go:618 store=3:3 2{/Table/11-/Table/12}: with replicaID [?], applying preemptive snapshot at index 24 (id=5684622a, encoded size=16, 1 rocksdb batches, 14 log entries) | |
| I160906 11:36:48.919244 1467 storage/replica_raftstorage.go:621 store=3:3 2{/Table/11-/Table/12}: with replicaID [?], applied preemptive snapshot in 0.002s | |
| I160906 11:36:48.926634 1124 storage/replica_command.go:1357 [n2,s2,r2{/Table/11-/Table/12}] 02489a94 pushed 8f6a0f68: pushee is expired | |
| I160906 11:36:48.938069 985 storage/replica_command.go:1357 [n1,s1,r2{/Table/11-/Table/12}] 02489a94 pushed 8f6a0f68: pushee is expired | |
| W160906 11:36:48.956698 1137 storage/store.go:2089 [n2,s2] conflicting intents on [/Local/Range/"\x93"/RangeDescriptor]: resolved? false | |
| I160906 11:36:48.957831 1137 storage/replica_command.go:3058 change replicas of 2: read existing descriptor range_id:2 start_key:"\223" end_key:"\224" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:3 | |
| I160906 11:36:48.966750 984 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/12,/Min), pErr=conflicting intents on [/Meta2/Table/12]: resolved? false | |
| I160906 11:36:48.970566 1246 storage/replica.go:2331 [n3,s3,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/12,/Min), pErr=conflicting intents on [/Meta2/Table/12]: resolved? false | |
| E160906 11:36:49.004148 1246 storage/replica.go:2272 [n3,s3,r1{/Min-/Table/11}] error executing raft command: conflicting intents on [/Meta2/Table/12]: resolved? false | |
| I160906 11:36:49.006542 1125 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/12,/Min), pErr=conflicting intents on [/Meta2/Table/12]: resolved? false | |
| E160906 11:36:49.011531 1125 storage/replica.go:2272 [n2,s2,r1{/Min-/Table/11}] error executing raft command: conflicting intents on [/Meta2/Table/12]: resolved? false | |
| I160906 11:36:49.013908 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.39 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 37ms/0s/37ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:36:49.016710 1104 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| W160906 11:36:49.031246 1469 storage/store.go:2089 [n1,s1] conflicting intents on [/Meta2/Table/12]: resolved? false | |
| I160906 11:36:49.054240 983 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/12,/Min), pErr=<nil> | |
| I160906 11:36:49.063197 1246 storage/replica.go:2331 [n3,s3,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/12,/Min), pErr=<nil> | |
| I160906 11:36:49.070104 1126 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Table/12,/Min), pErr=<nil> | |
| I160906 11:36:49.080233 1125 storage/replica.go:2331 [n2,s2,r2{/Table/11-/Table/12}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x93"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:49.082695 983 storage/replica.go:2331 [n1,s1,r2{/Table/11-/Table/12}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x93"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:49.086239 1137 storage/log.go:85 Range Event: "add", range: 2, info: {"AddReplica":{"node_id":3,"store_id":3,"replica_id":3},"UpdatedDesc":{"range_id":2,"start_key":"kw==","end_key":"lA==","replicas":[{"node_id":1,"store_id":1,"replica_id":1},{"node_id":2,"store_id":2,"replica_id":2},{"node_id":3,"store_id":3,"replica_id":3}],"next_replica_id":4}} | |
| I160906 11:36:49.130696 1137 kv/dist_sender.go:1036 application error: range 4: replica {2 2 3} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:49.136800 1137 kv/dist_sender.go:1107 [n2] range 4: caching new lease holder {NodeID:1 StoreID:1 ReplicaID:1} | |
| I160906 11:36:49.167359 985 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/6/1,/Min), pErr=<nil> | |
| I160906 11:36:49.173050 1126 storage/replica.go:2331 [n2,s2,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/6/1,/Min), pErr=<nil> | |
| I160906 11:36:49.176798 1244 storage/replica.go:2331 [n3,s3,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:48.920675Z/173969929431777280/6/1,/Min), pErr=<nil> | |
| I160906 11:36:49.184658 1137 storage/replica.go:1600 [n2,s2,r2{/Table/11-/Table/12}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:3} for range 2: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:3}] | |
| I160906 11:36:49.197790 1124 storage/replica.go:2331 [n2,s2,r2{/Table/11-/Table/12}] applied part of replica change txn: EndTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:49.205491 985 storage/replica.go:2331 [n1,s1,r2{/Table/11-/Table/12}] applied part of replica change txn: EndTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:49.230451 1493 kv/dist_sender.go:1036 application error: end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| I160906 11:36:49.238190 1493 kv/dist_sender.go:771 reply error GC [/Local/Range/"\x93"/RangeDescriptor,/Local/Range/"\x93"/"rdsc\x00"): end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| I160906 11:36:49.238350 1493 internal/client/db.go:498 failed batch: end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| W160906 11:36:49.238428 1493 storage/intent_resolver.go:344 could not GC completed transaction: end key /Local/Range/"\x93"/"rdsc\x00" must be greater than start /Local/Range/"\x93"/RangeDescriptor | |
| I160906 11:36:49.325575 1148 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:49.327685 1188 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:49.411907 1505 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:36:49.416521 1247 storage/replica_command.go:1357 [n3,s3,r2{/Table/11-/Table/12}] 02489a94 pushed 8f6a0f68: pushee is expired | |
| I160906 11:36:49.418447 1247 storage/replica.go:2331 [n3,s3,r2{/Table/11-/Table/12}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x93"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:49.419309 1247 storage/replica.go:2331 [n3,s3,r2{/Table/11-/Table/12}] applied part of replica change txn: EndTransaction [/Local/Range/"\x93"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:49.743219 1233 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:50.265879 1317 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:50.555478 1429 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:50.969645 1001 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:51.249976 1438 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:51.608339 1058 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:52.038404 1104 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:52.339756 1148 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:52.341848 1188 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:52.528109 948 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 2: 0 3 0 10.3 0.0 0.0 10.4 | |
| 3: 0 1 0 5.9 0.0 0.0 5.9 | |
| I160906 11:36:52.600626 1014 server/status/runtime.go:207 runtime stats: 303 MiB RSS, 238 goroutines, 25 MiB/1.4 MiB/35 MiB GO alloc/idle/total, 72 MiB/85 MiB CGO alloc/total, 461.67cgo/sec, 0.87/0.01 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:36:52.646728 947 kv/txn_coord_sender.go:251 [n1] txn coordinator: 1.60 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 129ms/46ms/223ms avg/σ/max duration, 0.1/0.3/1 avg/σ/max restarts (13 samples) | |
| I160906 11:36:52.789616 1233 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:53.305540 1317 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:53.471962 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 10.1 0.0 0.0 9.9 | |
| 3: 0 1 0 0.8 0.0 0.0 0.7 | |
| I160906 11:36:53.628276 1429 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:53.702898 1094 server/status/runtime.go:207 runtime stats: 311 MiB RSS, 238 goroutines, 28 MiB/1.5 MiB/37 MiB GO alloc/idle/total, 72 MiB/85 MiB CGO alloc/total, 449.46cgo/sec, 0.86/0.01 %(u/s)time, 0.00 %gc (2x) | |
| I160906 11:36:53.757334 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.33 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 158ms/84ms/244ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (2 samples) | |
| I160906 11:36:53.884396 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 5.7 0.0 0.0 5.8 | |
| 2: 0 1 0 0.8 0.0 0.0 0.8 | |
| I160906 11:36:54.036991 1001 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:54.042842 983 storage/replica_trigger.go:330 [n1,s1,r5{/Table/14-/Max}] range 5: replicaID 1 transfer raft leadership to replicaID 2 | |
| I160906 11:36:54.066568 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.28 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 37ms/0s/37ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:36:54.086816 1126 storage/replica_trigger.go:308 [n2,s2,r5{/Table/14-/Max}] new range lease replica {2 2 2} 2016-09-06 11:36:41.571292356 +0000 UTC 21.424812558s following replica {1 1 1} 1970-01-01 00:00:00 +0000 UTC 409211h36m41.571292356s [physicalTime=2016-09-06 11:36:54.086555139 +0000 UTC] | |
| I160906 11:36:54.089177 1137 storage/replicate_queue.go:154 store=2:2 5{/Table/14-/Max}: adding replica to {NodeID:3 StoreID:3 ReplicaID:0} due to under-replication | |
| I160906 11:36:54.134927 1536 storage/reservation.go:197 new reservation added: header:<node_id:3 store_id:3 > from_node_id:2 from_store_id:2 range_id:5 | |
| I160906 11:36:54.152893 1537 storage/replica_raftstorage.go:488 generated snapshot 5b2a7b58-f820-410d-b61d-da43e6b8ffea for range 5 at index 15 in 277.48µs. 0 log entries | |
| I160906 11:36:54.210198 1137 storage/store.go:2634 streamed snapshot: kv pairs: 10, log entries: 5 | |
| I160906 11:36:54.231348 1555 storage/replica_raftstorage.go:618 store=3:3 5{/Table/14-/Max}: with replicaID [?], applying preemptive snapshot at index 15 (id=5b2a7b58, encoded size=16, 1 rocksdb batches, 5 log entries) | |
| I160906 11:36:54.253183 1555 storage/replica_raftstorage.go:621 store=3:3 5{/Table/14-/Max}: with replicaID [?], applied preemptive snapshot in 0.022s | |
| I160906 11:36:54.278503 1137 storage/replica_command.go:3058 change replicas of 5: read existing descriptor range_id:5 start_key:"\226" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:3 | |
| I160906 11:36:54.355527 1126 storage/replica.go:2237 [n2,s2,r5{/Table/14-/Max}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:54.356448 1126 storage/replica.go:2272 [n2,s2,r5{/Table/14-/Max}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:54.383156 1438 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:54.383272 985 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Max,/Min), pErr=conflicting intents on [/Meta2/Max]: resolved? false | |
| I160906 11:36:54.385894 1558 kv/dist_sender.go:1036 application error: range 5: replica {1 1 1} not lease holder; node_id:2 store_id:2 replica_id:2 is | |
| I160906 11:36:54.386043 1558 kv/dist_sender.go:1107 [n1] range 5: caching new lease holder {NodeID:2 StoreID:2 ReplicaID:2} | |
| I160906 11:36:54.386395 1558 kv/dist_sender.go:771 reply error PushTxn [/Local/Range/"\x96"/RangeDescriptor,/Min): failed to send RPC: sending to all 1 replicas failed; last error: range 5: replica {1 1 1} not lease holder; node_id:2 store_id:2 replica_id:2 is | |
| I160906 11:36:54.420258 1223 server/status/runtime.go:207 runtime stats: 311 MiB RSS, 242 goroutines, 26 MiB/2.0 MiB/37 MiB GO alloc/idle/total, 73 MiB/85 MiB CGO alloc/total, 413.87cgo/sec, 0.86/0.01 %(u/s)time, 0.00 %gc (2x) | |
| I160906 11:36:54.432203 1246 storage/replica.go:2331 [n3,s3,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Max,/Min), pErr=conflicting intents on [/Meta2/Max]: resolved? false | |
| E160906 11:36:54.436066 1246 storage/replica.go:2272 [n3,s3,r1{/Min-/Table/11}] error executing raft command: conflicting intents on [/Meta2/Max]: resolved? false | |
| I160906 11:36:54.436631 984 storage/replica.go:2237 [n1,s1,r5{/Table/14-/Max}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:54.437541 984 storage/replica.go:2272 [n1,s1,r5{/Table/14-/Max}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:54.456117 1125 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Max,/Min), pErr=conflicting intents on [/Meta2/Max]: resolved? false | |
| E160906 11:36:54.456464 1125 storage/replica.go:2272 [n2,s2,r1{/Min-/Table/11}] error executing raft command: conflicting intents on [/Meta2/Max]: resolved? false | |
| I160906 11:36:54.497372 1541 storage/intent_resolver.go:194 skipping PushTxn for f6568178-8760-4c7a-835f-639596ed88ee; attempt already in flight | |
| I160906 11:36:54.686765 1058 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| W160906 11:36:54.732122 1558 storage/store.go:2089 [n1,s1] conflicting intents on [/Meta2/Max]: resolved? false | |
| I160906 11:36:54.780667 985 storage/replica.go:2331 [n1,s1,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Max,/Min), pErr=<nil> | |
| I160906 11:36:54.794284 1125 storage/replica.go:2331 [n2,s2,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Max,/Min), pErr=<nil> | |
| I160906 11:36:54.817230 1247 storage/replica.go:2331 [n3,s3,r1{/Min-/Table/11}] applied part of replica change txn: Noop [/Min,/Min), Noop [/Min,/Min), Put [/Meta2/Max,/Min), pErr=<nil> | |
| I160906 11:36:54.861897 1127 storage/replica.go:2331 [n2,s2,r5{/Table/14-/Max}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x96"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x96"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:54.863438 1137 storage/log.go:85 Range Event: "add", range: 5, info: {"AddReplica":{"node_id":3,"store_id":3,"replica_id":3},"UpdatedDesc":{"range_id":5,"start_key":"lg==","end_key":"//8=","replicas":[{"node_id":1,"store_id":1,"replica_id":1},{"node_id":2,"store_id":2,"replica_id":2},{"node_id":3,"store_id":3,"replica_id":3}],"next_replica_id":4}} | |
| I160906 11:36:54.912481 983 storage/replica.go:2331 [n1,s1,r5{/Table/14-/Max}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x96"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x96"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:55.061450 983 storage/replica.go:2331 [n1,s1,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/6/1,/Min), pErr=<nil> | |
| I160906 11:36:55.066020 1245 storage/replica.go:2331 [n3,s3,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/6/1,/Min), pErr=<nil> | |
| I160906 11:36:55.069520 1125 storage/replica.go:2331 [n2,s2,r4{/Table/13-/Table/14}] applied part of replica change txn: ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/0,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/2/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/3/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/4/1,/Min), ConditionalPut [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/6/1,/Min), pErr=<nil> | |
| I160906 11:36:55.072595 1137 storage/replica.go:1600 [n2,s2,r5{/Table/14-/Max}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:3} for range 5: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:3}] | |
| I160906 11:36:55.076329 1124 storage/replica.go:2331 [n2,s2,r5{/Table/14-/Max}] applied part of replica change txn: EndTransaction [/Local/Range/"\x96"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:55.114026 984 storage/replica.go:2331 [n1,s1,r5{/Table/14-/Max}] applied part of replica change txn: EndTransaction [/Local/Range/"\x96"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:55.216654 1104 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:55.264409 927 util/stop/stopper.go:342 stop has been called, stopping or quiescing all running tasks | |
| I160906 11:36:55.264505 927 util/stop/stopper.go:342 stop has been called, stopping or quiescing all running tasks | |
| I160906 11:36:55.264666 927 util/stop/stopper.go:414 quiescing; tasks left: | |
| 1 server/node.go:835 | |
| I160906 11:36:55.266934 929 rpc/context.go:169 closing 127.0.0.1:56887 | |
| I160906 11:36:55.267231 929 rpc/context.go:169 closing 127.0.0.1:41417 | |
| I160906 11:36:55.267527 929 rpc/context.go:169 closing 127.0.0.1:59236 | |
| W160906 11:36:55.269113 1300 storage/raft_transport.go:410 raft transport stream to node 2 failed: rpc error: code = 13 desc = transport is closing | |
| W160906 11:36:55.269678 1344 storage/raft_transport.go:410 raft transport stream to node 3 failed: rpc error: code = 13 desc = transport is closing | |
| I160906 11:36:55.277501 1577 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:56887 | |
| W160906 11:36:55.279114 1365 storage/raft_transport.go:410 raft transport stream to node 1 failed: EOF | |
| I160906 11:36:55.280243 1577 gossip/client.go:189 [n3] node 3: sending [sentinel first-range cluster-id store:2 system-db store:1 node:3 store:3 node:1 node:2] | |
| I160906 11:36:55.280903 1577 gossip/client.go:189 [n3] node 3: sending [cluster-id node:2 store:1 store:2 system-db store:3 first-range node:1 sentinel node:3] | |
| W160906 11:36:55.295422 1305 storage/raft_transport.go:410 raft transport stream to node 1 failed: EOF | |
| I160906 11:36:55.297307 1590 gossip/client.go:125 [n2] node 2: started gossip client to 127.0.0.1:56887 | |
| I160906 11:36:55.299645 1590 gossip/client.go:189 [n2] node 2: sending [node:2 store:2 node:3 store:3 store:1 first-range cluster-id node:1 sentinel system-db] | |
| I160906 11:36:55.300311 1590 gossip/client.go:189 [n2] node 2: sending [node:1 cluster-id store:2 node:3 sentinel store:1 system-db first-range store:3 node:2] | |
| I160906 11:36:55.320928 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| I160906 11:36:55.322239 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.337301 1225 kv/dist_sender.go:1036 application error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.337625 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.365848 1148 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:55.391743 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| I160906 11:36:55.392928 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.396789 1225 kv/dist_sender.go:1036 application error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.397090 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.505218 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| I160906 11:36:55.506423 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.510187 1225 kv/dist_sender.go:1036 application error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.510485 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.689906 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| I160906 11:36:55.691248 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.695816 1225 kv/dist_sender.go:1036 application error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.696224 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:55.799150 1233 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:56.008911 1631 storage/raft_transport.go:405 raft transport stream to node 1 established | |
| I160906 11:36:56.015384 1245 storage/replica.go:2237 [n3,s3,r5{/Table/14-/Max}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:56.016443 1245 storage/replica.go:2272 [n3,s3,r5{/Table/14-/Max}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:56.017958 1245 storage/replica.go:2331 [n3,s3,r5{/Table/14-/Max}] applied part of replica change txn: BeginTransaction [/Local/Range/"\x96"/RangeDescriptor,/Min), ConditionalPut [/Local/Range/"\x96"/RangeDescriptor,/Min), Noop [/Min,/Min), pErr=<nil> | |
| I160906 11:36:56.018967 1245 storage/replica.go:2331 [n3,s3,r5{/Table/14-/Max}] applied part of replica change txn: EndTransaction [/Local/Range/"\x96"/RangeDescriptor,/Min), pErr=<nil> | |
| I160906 11:36:56.162015 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| I160906 11:36:56.163084 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:56.166680 1225 kv/dist_sender.go:1036 application error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:56.166942 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:56.278233 1651 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:36:56.280125 1651 gossip/client.go:189 [n3] node 3: sending [store:3 cluster-id sentinel store:1 node:3 first-range node:1 system-db store:2 node:2] | |
| I160906 11:36:56.280732 1651 gossip/client.go:189 [n3] node 3: sending [system-db node:1 cluster-id store:2 store:3 first-range node:2 sentinel store:1 node:3] | |
| I160906 11:36:56.283063 1665 gossip/server.go:289 [n2] node 2: received [] from node 3 (0 fresh) | |
| I160906 11:36:56.284402 1665 gossip/server.go:289 [n2] node 2: received [cluster-id node:1 node:2 store:1 first-range node:3 sentinel store:2 store:3 system-db] from node 3 (2 fresh) | |
| I160906 11:36:56.285005 1665 gossip/server.go:289 [n2] node 2: received [first-range node:1 node:2 system-db cluster-id node:3 sentinel store:1 store:2 store:3] from node 3 (0 fresh) | |
| I160906 11:36:56.297898 1668 gossip/client.go:125 [n2] node 2: started gossip client to 127.0.0.1:59236 | |
| I160906 11:36:56.299613 1668 gossip/client.go:189 [n2] node 2: sending [node:3 store:3 cluster-id sentinel store:1 system-db store:2 first-range node:1 node:2] | |
| I160906 11:36:56.300265 1668 gossip/client.go:189 [n2] node 2: sending [first-range sentinel node:1 store:1 system-db store:2 node:3 store:3 cluster-id node:2] | |
| I160906 11:36:56.321253 1639 gossip/server.go:289 [n3] node 3: received [] from node 2 (0 fresh) | |
| I160906 11:36:56.325755 1639 gossip/server.go:289 [n3] node 3: received [cluster-id first-range node:1 node:3 sentinel store:1 store:3 node:2 store:2 system-db] from node 2 (2 fresh) | |
| I160906 11:36:56.337774 1639 gossip/server.go:289 [n3] node 3: received [node:2 node:3 store:2 system-db cluster-id first-range store:1 store:3 node:1 sentinel] from node 2 (0 fresh) | |
| I160906 11:36:56.630812 1429 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:57.046122 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| I160906 11:36:57.047471 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:57.052502 1225 kv/dist_sender.go:1036 application error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:57.053230 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: range 1: replica {2 2 2} not lease holder; node_id:1 store_id:1 replica_id:1 is | |
| I160906 11:36:57.361562 1161 gossip/gossip.go:1057 [n3] node has connected to cluster via gossip | |
| I160906 11:36:57.362302 1161 storage/stores.go:328 wrote 2 node addresses to persistent storage | |
| I160906 11:36:57.394786 1438 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:57.408791 1683 storage/raft_transport.go:405 raft transport stream to node 1 established | |
| I160906 11:36:57.674302 947 kv/txn_coord_sender.go:251 [n1] txn coordinator: 1.15 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 129ms/46ms/223ms avg/σ/max duration, 0.1/0.3/1 avg/σ/max restarts (13 samples) | |
| I160906 11:36:57.689516 1058 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:58.212555 1124 storage/replica.go:2237 [n2,s2,r1{/Min-/Table/11}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:58.213387 1124 storage/replica.go:2272 [n2,s2,r1{/Min-/Table/11}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:58.214581 1246 storage/replica.go:2237 [n3,s3,r1{/Min-/Table/11}] applying command with forced error: storage/replica.go:2162: no-op on empty Raft entry | |
| E160906 11:36:58.215452 1246 storage/replica.go:2272 [n3,s3,r1{/Min-/Table/11}] error executing raft command: storage/replica.go:2162: no-op on empty Raft entry | |
| I160906 11:36:58.219441 1104 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| W160906 11:36:58.247108 1572 kv/dist_sender.go:1052 RPC error: rpc error: code = 4 desc = context deadline exceeded | |
| I160906 11:36:58.251159 1317 rpc/context.go:169 closing 127.0.0.1:59236 | |
| I160906 11:36:58.251313 1188 rpc/context.go:169 closing 127.0.0.1:41417 | |
| I160906 11:36:58.251894 1151 http2_server.go:276 transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:56887->127.0.0.1:35231: use of closed network connection | |
| I160906 11:36:58.252123 1001 rpc/context.go:169 closing 127.0.0.1:56887 | |
| I160906 11:36:58.254368 1061 http2_server.go:276 transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:56887->127.0.0.1:35210: use of closed network connection | |
| I160906 11:36:58.254589 1173 http2_client.go:1053 transport: http2Client.notifyError got notified that the client transport was broken EOF. | |
| I160906 11:36:58.255316 1055 http2_client.go:1053 transport: http2Client.notifyError got notified that the client transport was broken EOF. | |
| I160906 11:36:58.256144 1024 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:36:58.256484 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:36:58.305956 1020 kv/transport_race.go:68 transport race promotion: ran 266 iterations on up to 277 requests | |
| I160906 11:36:58.306128 927 storage/engine/rocksdb.go:402 closing in-memory rocksdb instance | |
| I160906 11:36:58.307335 927 util/stop/stopper.go:342 stop has been called, stopping or quiescing all running tasks | |
| I160906 11:36:58.307586 927 util/stop/stopper.go:414 quiescing; tasks left: | |
| 1 storage/replica_range_lease.go:143 | |
| 1 storage/intent_resolver.go:346 | |
| 1 server/node.go:835 | |
| I160906 11:36:58.307694 1034 rpc/context.go:169 closing 127.0.0.1:41417 | |
| I160906 11:36:58.308014 1034 rpc/context.go:169 closing 127.0.0.1:59236 | |
| I160906 11:36:58.308407 1034 rpc/context.go:169 closing 127.0.0.1:56887 | |
| W160906 11:36:58.310999 1500 storage/raft_transport.go:410 raft transport stream to node 3 failed: rpc error: code = 13 desc = transport is closing | |
| I160906 11:36:58.311428 1024 /go/src/google.golang.org/grpc/clientconn.go:767 grpc: addrConn.transportMonitor exits due to: context canceled | |
| W160906 11:36:58.314612 1161 gossip/gossip.go:1043 [n3] no incoming or outgoing connections | |
| W160906 11:36:58.314818 1505 storage/raft_transport.go:410 raft transport stream to node 2 failed: EOF | |
| I160906 11:36:58.316503 1695 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:36:58.318508 1695 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:2 node:3 cluster-id store:2 node:1 sentinel store:1 system-db] | |
| I160906 11:36:58.319344 1695 gossip/client.go:189 [n3] node 3: sending [store:2 system-db store:3 node:1 node:2 sentinel store:1 cluster-id first-range node:3] | |
| I160906 11:36:58.786928 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.29 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 369ms/305ms/792ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (3 samples) | |
| I160906 11:36:58.801691 1233 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:36:58.863241 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:36:58.867941 1715 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:36:59.082632 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.20 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 37ms/0s/37ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:36:59.255783 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| W160906 11:36:59.316660 1719 gossip/client.go:119 [n3] node 3: failed to start gossip client: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:00.321441 1720 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:00.331599 1720 gossip/client.go:189 [n3] node 3: sending [node:2 sentinel store:1 node:3 cluster-id store:3 first-range node:1 store:2 system-db] | |
| I160906 11:37:00.402715 1438 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:37:00.764521 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:37:00.811694 1126 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonNewLeader | |
| I160906 11:37:01.248477 1572 kv/dist_sender.go:1036 application error: range 4: replica {2 2 3} not lease holder; <nil> is | |
| I160906 11:37:01.248859 927 util/stop/stopper.go:414 quiescing; tasks left: | |
| 1 storage/replica_range_lease.go:143 | |
| 1 storage/intent_resolver.go:346 | |
| W160906 11:37:01.249298 1572 kv/dist_sender.go:1052 RPC error: rpc error: code = 9 desc = grpc: the client connection is closing | |
| I160906 11:37:01.249977 1572 kv/dist_sender.go:771 reply error ResolveIntent [/Meta2/Max,/Min), ResolveIntent [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/0,/Min), ResolveIntent [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/2/1,/Min), ResolveIntent [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/3/1,/Min), ResolveIntent [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/4/1,/Min), ResolveIntent [/Table/13/1/2016-09-06T11:36:54.276869Z/173969948846653440/6/1,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 9 desc = grpc: the client connection is closing | |
| I160906 11:37:01.250421 1572 internal/client/db.go:498 failed batch: failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 9 desc = grpc: the client connection is closing | |
| W160906 11:37:01.250801 1572 storage/intent_resolver.go:325 store=2:2 5{/Table/14-/Max}: failed to resolve intents: failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 9 desc = grpc: the client connection is closing | |
| I160906 11:37:01.251006 927 util/stop/stopper.go:414 quiescing; tasks left: | |
| 1 storage/replica_range_lease.go:143 | |
| I160906 11:37:01.316647 1733 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:01.514252 1244 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonNewLeader | |
| I160906 11:37:01.804073 1233 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:37:01.866882 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:01.870890 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:01.871431 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:01.871616 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:02.323245 1756 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:02.324824 1756 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 store:1 cluster-id store:2 node:3 sentinel system-db node:2] | |
| I160906 11:37:02.325291 1756 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 first-range sentinel store:1 store:3 cluster-id store:2 system-db node:1] | |
| W160906 11:37:02.407511 1122 raft/raft.go:647 store=2:2 2{/Table/11-/Table/12}: 2 stepped down to follower since quorum is not active | |
| I160906 11:37:02.519080 1771 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:02.656257 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:02.664859 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:02.666237 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:02.666864 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:02.718405 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| W160906 11:37:03.007351 1122 raft/raft.go:647 store=2:2 5{/Table/14-/Max}: 2 stepped down to follower since quorum is not active | |
| I160906 11:37:03.406062 1438 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:37:03.440434 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 4.3 0.0 0.0 4.2 | |
| 3: 0 1 0 1.3 0.0 0.0 1.4 | |
| I160906 11:37:03.571317 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:37:03.643166 1094 server/status/runtime.go:207 runtime stats: 329 MiB RSS, 105 goroutines, 21 MiB/6.9 MiB/39 MiB GO alloc/idle/total, 46 MiB/60 MiB CGO alloc/total, 236.31cgo/sec, 0.71/0.03 %(u/s)time, 0.00 %gc (1x) | |
| W160906 11:37:03.826999 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:37:03.828173 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:37:03.848298 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.21 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 369ms/305ms/792ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (3 samples) | |
| I160906 11:37:03.869101 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 2.3 0.0 0.0 2.2 | |
| 2: 0 1 0 1.6 0.0 0.0 1.3 | |
| I160906 11:37:04.119499 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.14 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 37ms/0s/37ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| W160906 11:37:04.208429 1122 raft/raft.go:647 store=2:2 1{/Min-/Table/11}: 2 stepped down to follower since quorum is not active | |
| I160906 11:37:04.315804 1744 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:04.322633 1780 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:04.326683 1780 gossip/client.go:189 [n3] node 3: sending [store:2 system-db node:2 sentinel cluster-id first-range node:1 store:1 node:3 store:3] | |
| I160906 11:37:04.327538 1780 gossip/client.go:189 [n3] node 3: sending [store:3 store:2 system-db first-range node:1 node:2 sentinel store:1 cluster-id node:3] | |
| I160906 11:37:04.359086 1223 server/status/runtime.go:207 runtime stats: 334 MiB RSS, 105 goroutines, 22 MiB/6.1 MiB/39 MiB GO alloc/idle/total, 46 MiB/60 MiB CGO alloc/total, 187.55cgo/sec, 0.69/0.03 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:37:04.807619 1233 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:37:05.526088 1799 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:05.728855 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:05.731089 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:05.734717 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:05.736248 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:05.736674 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:05.850790 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:37:06.020369 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:06.315568 1806 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:06.332730 1826 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:06.334526 1826 gossip/client.go:189 [n3] node 3: sending [store:2 system-db store:3 node:2 cluster-id store:1 node:3 first-range node:1 sentinel] | |
| I160906 11:37:06.335177 1826 gossip/client.go:189 [n3] node 3: sending [store:1 cluster-id node:3 store:3 first-range node:1 node:2 sentinel store:2 system-db] | |
| I160906 11:37:06.419916 1438 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:37:07.318240 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:37:07.830244 1233 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:37:07.915017 1841 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:07.930290 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:07.943069 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:07.943799 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:07.945701 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:08.314985 1824 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:08.351820 1859 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:08.354003 1859 gossip/client.go:189 [n3] node 3: sending [cluster-id store:2 node:3 store:3 first-range node:1 node:2 sentinel system-db store:1] | |
| I160906 11:37:08.354678 1859 gossip/client.go:189 [n3] node 3: sending [first-range node:1 store:1 node:3 store:2 system-db store:3 node:2 sentinel cluster-id] | |
| I160906 11:37:08.371459 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:08.374568 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:08.375075 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:08.375264 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:08.722921 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:08.870068 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.15 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 369ms/305ms/792ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (3 samples) | |
| I160906 11:37:08.870899 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:08.873387 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:08.880637 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:08.880879 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:09.007379 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:09.073622 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:37:09.148854 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.10 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 37ms/0s/37ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:37:09.422375 1438 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:37:10.357183 1880 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:10.359285 1880 gossip/client.go:189 [n3] node 3: sending [cluster-id node:3 first-range node:2 store:2 system-db store:1 store:3 node:1 sentinel] | |
| I160906 11:37:10.359931 1880 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 sentinel store:3 system-db first-range store:1 node:3 cluster-id store:2] | |
| I160906 11:37:10.515320 1895 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:10.844145 1233 rpc/clock_offset.go:147 3 of 3 nodes are within the maximum offset of 50ms | |
| I160906 11:37:10.995472 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:11.076946 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:11.090508 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:11.117828 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:11.713386 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:11.915246 1904 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:12.007433 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:12.078371 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:12.080857 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:12.081350 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:12.081516 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:12.383999 1924 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:12.386090 1924 gossip/client.go:189 [n3] node 3: sending [node:2 store:1 node:3 cluster-id node:1 sentinel store:3 first-range store:2 system-db] | |
| I160906 11:37:12.386712 1924 gossip/client.go:189 [n3] node 3: sending [node:3 system-db first-range node:1 node:2 sentinel store:1 store:3 cluster-id store:2] | |
| I160906 11:37:12.425407 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:12.452262 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:37:13.439152 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:37:13.515425 1943 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:13.643259 1094 server/status/runtime.go:207 runtime stats: 349 MiB RSS, 105 goroutines, 14 MiB/14 MiB/40 MiB GO alloc/idle/total, 45 MiB/60 MiB CGO alloc/total, 37.70cgo/sec, 0.73/0.01 %(u/s)time, 0.00 %gc (1x) | |
| W160906 11:37:13.834452 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| W160906 11:37:13.834563 1095 ts/db.go:104 node unavailable; try another peer | |
| I160906 11:37:13.847140 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:13.868923 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 1 0 0.9 0.0 0.0 0.0 | |
| I160906 11:37:13.935023 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.11 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 369ms/305ms/792ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (3 samples) | |
| I160906 11:37:14.115122 1909 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:14.184808 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.07 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 37ms/0s/37ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:37:14.254288 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:14.261433 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:14.261978 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:14.263920 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:14.363929 1223 server/status/runtime.go:207 runtime stats: 349 MiB RSS, 105 goroutines, 15 MiB/13 MiB/40 MiB GO alloc/idle/total, 45 MiB/60 MiB CGO alloc/total, 37.48cgo/sec, 0.74/0.01 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:37:14.385637 1918 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:14.387998 1918 gossip/client.go:189 [n3] node 3: sending [store:3 sentinel store:1 node:3 cluster-id first-range node:1 node:2 store:2 system-db] | |
| I160906 11:37:14.388806 1918 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 first-range node:1 sentinel store:1 store:3 system-db cluster-id store:2] | |
| I160906 11:37:14.718648 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:15.007441 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:15.018918 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:37:15.427954 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:15.457963 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:15.460607 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:15.461134 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:15.461318 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:16.179220 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:16.182195 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:16.182740 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:16.182936 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:16.390579 1951 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:16.392481 1951 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id store:1 store:3 store:2 node:2 sentinel node:3 first-range node:1] | |
| I160906 11:37:16.393124 1951 gossip/client.go:189 [n3] node 3: sending [cluster-id store:1 node:3 first-range node:1 node:2 store:2 system-db sentinel store:3] | |
| I160906 11:37:16.715032 1998 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:16.849493 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:17.459186 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:17.466291 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:17.466827 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:17.468727 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:17.714476 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:17.716924 1973 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:18.007437 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:18.407469 1977 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:18.409418 1977 gossip/client.go:189 [n3] node 3: sending [first-range sentinel store:1 node:3 cluster-id store:2 system-db store:3 node:1 node:2] | |
| I160906 11:37:18.410076 1977 gossip/client.go:189 [n3] node 3: sending [store:1 store:2 node:3 node:1 sentinel node:2 cluster-id system-db store:3 first-range] | |
| I160906 11:37:18.430322 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:18.960122 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.08 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 369ms/305ms/792ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (3 samples) | |
| I160906 11:37:19.200256 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.05 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 37ms/0s/37ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:37:19.210026 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:19.212589 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:19.213134 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:19.213411 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:19.874646 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:19.915431 2009 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:20.409749 2013 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:20.411900 2013 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:2 cluster-id store:2 system-db store:1 node:3 sentinel node:1] | |
| I160906 11:37:20.412700 2013 gossip/client.go:189 [n3] node 3: sending [system-db node:1 sentinel node:3 store:3 store:2 first-range node:2 store:1 cluster-id] | |
| I160906 11:37:20.590254 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:37:20.713354 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:20.856869 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:20.864061 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:20.864628 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:20.866315 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:20.915101 2050 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:21.007527 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:21.436561 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:21.715081 2030 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:22.411176 2066 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:22.413354 2066 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range store:3 store:1 node:3 store:2 node:1 node:2 sentinel] | |
| I160906 11:37:22.413994 2066 gossip/client.go:189 [n3] node 3: sending [system-db sentinel node:3 node:2 cluster-id store:2 store:1 store:3 first-range node:1] | |
| I160906 11:37:22.877582 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:23.441555 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:37:23.515025 2083 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:23.592809 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:23.595428 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:23.595964 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:23.596172 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:23.643471 1094 server/status/runtime.go:207 runtime stats: 349 MiB RSS, 105 goroutines, 19 MiB/9.3 MiB/40 MiB GO alloc/idle/total, 45 MiB/60 MiB CGO alloc/total, 32.00cgo/sec, 0.73/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:37:23.713293 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:37:23.822722 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:37:23.822925 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:37:23.868896 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 1 0 0.7 0.0 0.0 0.0 | |
| I160906 11:37:23.992581 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.05 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 369ms/305ms/792ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (3 samples) | |
| I160906 11:37:24.007487 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:24.175512 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:24.178325 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:24.178899 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:24.179084 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:24.269899 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.04 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 37ms/0s/37ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:37:24.379260 1223 server/status/runtime.go:207 runtime stats: 349 MiB RSS, 105 goroutines, 20 MiB/8.8 MiB/40 MiB GO alloc/idle/total, 45 MiB/60 MiB CGO alloc/total, 30.55cgo/sec, 0.73/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:37:24.422377 2091 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:24.424205 2091 gossip/client.go:189 [n3] node 3: sending [store:1 node:3 first-range store:2 node:2 sentinel node:1 system-db cluster-id store:3] | |
| I160906 11:37:24.424871 2091 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 store:1 node:3 cluster-id store:2 sentinel store:3] | |
| I160906 11:37:24.438922 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:24.651214 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:24.661901 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:24.674468 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:24.676974 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:24.915364 2063 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:25.829797 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:37:25.879990 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:25.915083 2119 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:26.329630 2124 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:26.346633 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:37:26.443875 2130 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:26.446237 2130 gossip/client.go:189 [n3] node 3: sending [node:3 node:2 sentinel cluster-id system-db store:3 first-range node:1 store:1 store:2] | |
| I160906 11:37:26.447068 2130 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range system-db cluster-id node:2 sentinel store:1 node:1 store:2] | |
| I160906 11:37:26.713328 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:27.007682 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:27.441511 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:28.445681 2147 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:28.447779 2147 gossip/client.go:189 [n3] node 3: sending [node:2 store:1 store:3 store:2 first-range sentinel node:3 cluster-id system-db node:1] | |
| I160906 11:37:28.448521 2147 gossip/client.go:189 [n3] node 3: sending [node:1 sentinel node:3 first-range system-db cluster-id store:2 node:2 store:1 store:3] | |
| I160906 11:37:28.516366 2162 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:28.882511 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:29.035600 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.04 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 369ms/305ms/792ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (3 samples) | |
| I160906 11:37:29.285474 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.03 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 37ms/0s/37ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (1 samples) | |
| I160906 11:37:29.349082 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:29.351628 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:29.352197 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:29.352360 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:29.515992 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:29.524208 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:29.524917 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:29.527311 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:29.714994 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:29.716762 2183 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:30.009959 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:30.443995 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:30.446861 2168 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:30.448876 2168 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 node:2 store:2 system-db first-range node:1 sentinel store:1 cluster-id] | |
| I160906 11:37:30.449485 2168 gossip/client.go:189 [n3] node 3: sending [node:3 cluster-id store:2 store:3 node:1 store:1 system-db first-range node:2 sentinel] | |
| I160906 11:37:31.114952 2199 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:31.884905 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:32.448622 2205 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:32.450902 2205 gossip/client.go:189 [n3] node 3: sending [node:3 first-range node:1 node:2 cluster-id system-db sentinel store:1 store:3 store:2] | |
| I160906 11:37:32.451511 2205 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 sentinel cluster-id system-db node:3 store:2 store:1] | |
| I160906 11:37:32.687596 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:32.690412 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:32.690914 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:32.691096 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:32.715211 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:32.914973 2224 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:33.007392 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:33.314810 2186 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:33.439171 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:37:33.446389 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:33.618528 1092 gossip/gossip.go:422 [n2] gossip status (ok, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 42/37 sent/received, bytes 17181B/13485B sent/received) | |
| W160906 11:37:33.618939 1092 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| I160906 11:37:33.643195 1094 server/status/runtime.go:207 runtime stats: 349 MiB RSS, 105 goroutines, 15 MiB/13 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 36.60cgo/sec, 0.73/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:37:33.705453 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| W160906 11:37:33.822715 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:37:33.822939 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| W160906 11:37:33.842990 1130 storage/store.go:1034 [n2,s2] error gossiping system config: node unavailable; try another peer | |
| I160906 11:37:33.845856 1129 storage/replica.go:2845 [n2,s2,r1{/Min-/Table/11}] gossiping cluster id {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} from store 2, range 1 | |
| W160906 11:37:33.846108 1129 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| W160906 11:37:33.846289 1129 storage/store.go:1014 [n2,s2] error gossiping first range data: node unavailable; try another peer | |
| I160906 11:37:33.868867 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 1 0 0.8 0.0 0.0 0.0 | |
| I160906 11:37:34.065919 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.03 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 517ms/273ms/792ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (2 samples) | |
| I160906 11:37:34.287858 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.02 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:37:34.359038 1223 server/status/runtime.go:207 runtime stats: 349 MiB RSS, 105 goroutines, 15 MiB/12 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 35.77cgo/sec, 0.74/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:37:34.359819 1221 gossip/gossip.go:422 [n3] gossip status (ok, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 392/32 sent/received, bytes 163403B/12946B sent/received) | |
| I160906 11:37:34.450270 2228 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:34.452384 2228 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id store:1 sentinel store:2 system-db] | |
| I160906 11:37:34.453037 2228 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 store:1 store:2 node:3 first-range node:1 node:2 sentinel] | |
| I160906 11:37:34.548534 1249 storage/replica.go:2845 [n3,s3,r1{/Min-/Table/11}] gossiping cluster id {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} from store 3, range 1 | |
| I160906 11:37:34.887427 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:35.446764 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:35.456138 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:35.456694 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:35.458662 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:35.713339 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:36.007430 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:36.115392 2266 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:36.460552 2271 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:36.462280 2271 gossip/client.go:189 [n3] node 3: sending [node:2 sentinel node:3 store:3 node:1 store:1 first-range system-db cluster-id store:2] | |
| I160906 11:37:36.462940 2271 gossip/client.go:189 [n3] node 3: sending [system-db store:1 node:3 sentinel cluster-id store:2 store:3 first-range node:1 node:2] | |
| I160906 11:37:36.466474 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:36.707814 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:36.710344 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:36.710893 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:36.711078 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:37.672059 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:37:37.889899 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:38.461568 2295 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:38.463851 2295 gossip/client.go:189 [n3] node 3: sending [cluster-id store:2 system-db store:1 store:3 node:1 sentinel node:3 first-range node:2] | |
| I160906 11:37:38.464564 2295 gossip/client.go:189 [n3] node 3: sending [system-db node:3 node:2 sentinel node:1 cluster-id store:2 store:1 store:3 first-range] | |
| I160906 11:37:38.714063 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:39.007375 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:39.091451 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.02 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 517ms/273ms/792ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (2 samples) | |
| I160906 11:37:39.289301 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.01 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:37:39.468948 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:40.463241 2243 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:40.465043 2243 gossip/client.go:189 [n3] node 3: sending [node:2 store:1 cluster-id node:3 node:1 first-range sentinel store:2 system-db store:3] | |
| I160906 11:37:40.465721 2243 gossip/client.go:189 [n3] node 3: sending [sentinel node:3 first-range system-db store:1 store:3 node:1 node:2 cluster-id store:2] | |
| I160906 11:37:40.675600 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:40.678496 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:40.679020 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:40.679230 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:40.681921 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:37:40.724847 2327 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:40.892415 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:41.538480 2313 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:41.713563 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:41.914728 2318 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:42.018047 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:42.119440 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:37:42.483912 2339 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:42.507274 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:43.439284 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:37:43.643083 1094 server/status/runtime.go:207 runtime stats: 349 MiB RSS, 105 goroutines, 19 MiB/9.2 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 33.30cgo/sec, 0.81/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:37:43.686383 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:43.693820 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:43.694416 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:43.696340 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| W160906 11:37:43.832265 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:37:43.832394 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:37:43.869289 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 1 0 0.4 0.0 0.0 0.0 | |
| I160906 11:37:43.895091 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:44.111936 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.01 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 517ms/273ms/792ms avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (2 samples) | |
| I160906 11:37:44.290946 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.01 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:37:44.292017 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:44.294719 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:44.295240 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:44.295429 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:44.359354 1223 server/status/runtime.go:207 runtime stats: 349 MiB RSS, 105 goroutines, 20 MiB/8.7 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 32.90cgo/sec, 0.79/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:37:44.476029 2359 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:44.477941 2359 gossip/client.go:189 [n3] node 3: sending [sentinel store:1 system-db store:3 first-range node:1 node:2 cluster-id store:2 node:3] | |
| I160906 11:37:44.478596 2359 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 node:3 store:3 cluster-id store:2 sentinel store:1] | |
| I160906 11:37:44.717909 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:44.736232 2374 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:45.014685 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:45.509929 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:45.915006 2393 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:45.983969 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:37:45.987817 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:37:46.477674 2401 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:46.479579 2401 gossip/client.go:189 [n3] node 3: sending [node:1 sentinel store:1 store:3 cluster-id node:2 node:3 first-range store:2 system-db] | |
| I160906 11:37:46.480267 2401 gossip/client.go:189 [n3] node 3: sending [node:1 cluster-id sentinel node:3 first-range store:2 system-db store:1 store:3 node:2] | |
| I160906 11:37:46.715433 2415 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:46.897645 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:47.713360 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:48.007662 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:48.479433 2376 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:48.481328 2376 gossip/client.go:189 [n3] node 3: sending [store:2 node:2 first-range store:3 system-db cluster-id node:1 sentinel store:1 node:3] | |
| I160906 11:37:48.482010 2376 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:1 sentinel node:3 store:2 first-range node:2 store:1 store:3] | |
| I160906 11:37:48.512580 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:48.919230 2421 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:48.990709 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:48.993706 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:48.994232 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:48.994421 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:49.113665 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.01 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:37:49.300661 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.01 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:37:49.515102 2432 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:49.900279 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:50.500002 2439 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:50.505318 2439 gossip/client.go:189 [n3] node 3: sending [node:3 cluster-id store:2 store:3 first-range node:2 sentinel system-db node:1 store:1] | |
| I160906 11:37:50.505991 2439 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range sentinel store:1 store:2 node:1] | |
| I160906 11:37:50.713396 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:51.007396 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:51.515469 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:51.858103 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:51.865218 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:51.865799 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:51.867751 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:52.401380 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:52.404447 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:52.405138 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:52.405384 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:52.487808 2472 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:52.489623 2472 gossip/client.go:189 [n3] node 3: sending [system-db store:3 first-range store:1 cluster-id store:2 node:1 node:2 sentinel node:3] | |
| I160906 11:37:52.490256 2472 gossip/client.go:189 [n3] node 3: sending [node:2 system-db store:3 first-range store:2 cluster-id node:1 sentinel store:1 node:3] | |
| I160906 11:37:52.515088 2487 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:52.909611 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:53.317276 2446 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:53.439176 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:37:53.643211 1094 server/status/runtime.go:207 runtime stats: 350 MiB RSS, 105 goroutines, 13 MiB/14 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 38.00cgo/sec, 0.73/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:37:53.713695 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:37:53.822712 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:37:53.822938 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:37:53.874122 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.8 0.0 0.0 0.0 | |
| I160906 11:37:54.007420 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:54.115241 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.01 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:37:54.302144 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.01 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:37:54.314944 2493 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:54.360931 1223 server/status/runtime.go:207 runtime stats: 350 MiB RSS, 105 goroutines, 14 MiB/14 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 38.79cgo/sec, 0.73/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:37:54.489815 2500 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:54.491909 2500 gossip/client.go:189 [n3] node 3: sending [store:2 store:3 first-range node:1 node:2 system-db store:1 node:3 sentinel cluster-id] | |
| I160906 11:37:54.492609 2500 gossip/client.go:189 [n3] node 3: sending [store:1 store:3 first-range cluster-id store:2 node:2 sentinel node:3 node:1 system-db] | |
| I160906 11:37:54.517985 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:55.912130 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:56.491596 2497 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:56.494006 2497 gossip/client.go:189 [n3] node 3: sending [first-range node:3 system-db store:2 node:1 node:2 sentinel store:1 store:3 cluster-id] | |
| I160906 11:37:56.494846 2497 gossip/client.go:189 [n3] node 3: sending [system-db store:3 store:1 cluster-id sentinel node:3 store:2 first-range node:1 node:2] | |
| I160906 11:37:56.714651 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:37:56.915016 2544 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:57.007450 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:37:57.200905 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:57.204048 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:57.204641 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:57.204824 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:57.274344 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:37:57.530376 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:58.493009 2556 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:37:58.494286 2556 gossip/client.go:189 [n3] node 3: sending [store:1 store:3 first-range node:1 cluster-id node:2 sentinel node:3 system-db store:2] | |
| I160906 11:37:58.494753 2556 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 system-db cluster-id store:2 store:3 sentinel store:1 node:3] | |
| I160906 11:37:58.515067 2571 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:37:58.914772 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:37:59.116826 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.01 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:37:59.303661 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:37:59.673413 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:37:59.681708 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:37:59.682393 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:59.684664 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:37:59.713302 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:00.007397 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:00.280994 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:00.283415 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:00.283905 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:00.284060 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:00.495033 2584 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:00.496955 2584 gossip/client.go:189 [n3] node 3: sending [node:1 sentinel store:1 node:3 store:3 first-range cluster-id node:2 store:2 system-db] | |
| I160906 11:38:00.497615 2584 gossip/client.go:189 [n3] node 3: sending [store:2 store:1 node:1 system-db cluster-id node:2 sentinel node:3 store:3 first-range] | |
| I160906 11:38:00.533016 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:01.655018 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:01.658532 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:01.914959 2527 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:01.919373 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:02.115159 2613 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:02.509857 2617 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:02.511727 2617 gossip/client.go:189 [n3] node 3: sending [first-range node:2 store:1 store:3 store:2 system-db cluster-id node:1 sentinel node:3] | |
| I160906 11:38:02.512477 2617 gossip/client.go:189 [n3] node 3: sending [node:2 sentinel store:1 node:3 system-db cluster-id store:3 first-range node:1 store:2] | |
| I160906 11:38:02.713314 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:03.007374 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:03.115138 2632 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:03.439160 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:38:03.535655 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:03.644980 1094 server/status/runtime.go:207 runtime stats: 351 MiB RSS, 105 goroutines, 18 MiB/10 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 36.09cgo/sec, 0.68/0.00 %(u/s)time, 0.00 %gc (0x) | |
| W160906 11:38:03.825067 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:38:03.825211 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:38:03.872458 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.7 0.0 0.0 0.0 | |
| I160906 11:38:04.118670 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:04.305379 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:04.359008 1223 server/status/runtime.go:207 runtime stats: 351 MiB RSS, 105 goroutines, 18 MiB/10 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 33.81cgo/sec, 0.67/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:38:04.511631 2638 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:04.513968 2638 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 store:2 system-db store:1 node:3 node:2 sentinel cluster-id] | |
| I160906 11:38:04.514762 2638 gossip/client.go:189 [n3] node 3: sending [node:3 store:2 node:1 node:2 sentinel store:1 cluster-id system-db store:3 first-range] | |
| I160906 11:38:04.661082 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:04.664072 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:04.664625 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:04.664811 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:04.921866 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:05.042581 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:05.515274 2661 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:05.713342 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:06.008039 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:06.513676 2665 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:06.515457 2665 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 sentinel store:1 cluster-id system-db store:2 node:3] | |
| I160906 11:38:06.516244 2665 gossip/client.go:189 [n3] node 3: sending [store:3 store:1 node:3 cluster-id store:2 first-range node:1 node:2 sentinel system-db] | |
| I160906 11:38:06.540538 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:06.915081 2680 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:07.915385 2687 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:07.924238 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:08.048212 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:08.050754 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:08.051251 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:08.051428 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:08.256365 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:08.263819 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:08.264455 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:08.266438 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:08.515204 2702 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:08.517276 2702 gossip/client.go:189 [n3] node 3: sending [cluster-id node:2 sentinel node:1 store:1 store:2 system-db node:3 store:3 first-range] | |
| I160906 11:38:08.517951 2702 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 store:1 cluster-id system-db first-range sentinel node:3 store:3 store:2] | |
| I160906 11:38:08.713286 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:09.007381 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:09.120211 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:09.306834 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:09.543075 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:09.656177 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:09.659509 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:10.515689 2722 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:10.518896 2727 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:10.520700 2727 gossip/client.go:189 [n3] node 3: sending [node:2 sentinel store:1 node:3 store:3 first-range system-db cluster-id store:2 node:1] | |
| I160906 11:38:10.521360 2727 gossip/client.go:189 [n3] node 3: sending [cluster-id store:3 first-range node:2 store:1 store:2 system-db node:1 sentinel node:3] | |
| I160906 11:38:10.926761 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:11.508838 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:38:11.713902 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:11.715718 2745 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:12.007398 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:12.518351 2600 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:12.522269 2604 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:12.524035 2604 gossip/client.go:189 [n3] node 3: sending [system-db store:2 node:2 sentinel store:1 node:3 cluster-id store:3 first-range node:1] | |
| I160906 11:38:12.524743 2604 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 sentinel cluster-id system-db store:1 store:2 node:2] | |
| I160906 11:38:12.554378 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:12.661919 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:12.664862 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:12.665381 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:12.665559 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:12.914813 2749 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:13.439209 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:38:13.643148 1094 server/status/runtime.go:207 runtime stats: 351 MiB RSS, 105 goroutines, 23 MiB/6.4 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.51cgo/sec, 0.65/0.00 %(u/s)time, 0.00 %gc (0x) | |
| W160906 11:38:13.822715 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:38:13.822947 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:38:13.868931 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.7 0.0 0.0 0.0 | |
| I160906 11:38:13.929232 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:14.121783 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:14.312396 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:14.359004 1223 server/status/runtime.go:207 runtime stats: 351 MiB RSS, 105 goroutines, 23 MiB/6.0 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 32.90cgo/sec, 0.65/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:38:14.523945 2770 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:14.525980 2770 gossip/client.go:189 [n3] node 3: sending [system-db store:3 node:1 node:2 sentinel cluster-id node:3 first-range store:1 store:2] | |
| I160906 11:38:14.526626 2770 gossip/client.go:189 [n3] node 3: sending [store:3 store:2 cluster-id node:1 sentinel node:3 first-range system-db node:2 store:1] | |
| I160906 11:38:14.726896 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:15.007397 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:15.322267 2769 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:15.557008 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:16.406971 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:16.409699 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:16.410196 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:16.410362 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:16.490836 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:16.499176 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:16.499862 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:16.512874 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:16.550629 2784 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:16.560674 2784 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:3 cluster-id store:2 store:1 store:3 node:2 sentinel] | |
| I160906 11:38:16.561703 2784 gossip/client.go:189 [n3] node 3: sending [sentinel store:1 node:3 node:2 store:3 first-range node:1 system-db cluster-id store:2] | |
| I160906 11:38:16.938277 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:17.513296 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:17.518750 2817 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:17.713383 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:18.007434 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:18.514798 2800 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:18.556223 2836 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:18.557993 2836 gossip/client.go:189 [n3] node 3: sending [sentinel store:2 first-range node:2 store:1 node:3 cluster-id system-db store:3 node:1] | |
| I160906 11:38:18.558633 2836 gossip/client.go:189 [n3] node 3: sending [sentinel store:1 node:3 store:2 node:2 store:3 first-range node:1 cluster-id system-db] | |
| I160906 11:38:18.563525 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:19.123583 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:19.314213 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:19.916331 2823 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:19.950809 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:20.516976 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:20.519776 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:20.520411 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:20.520587 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:20.557918 2859 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:20.559685 2859 gossip/client.go:189 [n3] node 3: sending [store:2 system-db store:3 node:2 store:1 cluster-id node:3 first-range node:1 sentinel] | |
| I160906 11:38:20.560576 2859 gossip/client.go:189 [n3] node 3: sending [store:1 sentinel store:2 node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:38:20.713350 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:21.007396 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:21.566015 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:21.798210 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:21.801532 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:22.559497 2879 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:22.561793 2879 gossip/client.go:189 [n3] node 3: sending [sentinel store:2 node:1 first-range node:2 store:1 node:3 store:3 cluster-id system-db] | |
| I160906 11:38:22.562461 2879 gossip/client.go:189 [n3] node 3: sending [node:1 sentinel node:3 store:3 system-db node:2 store:1 first-range store:2 cluster-id] | |
| I160906 11:38:22.715401 2894 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:22.915323 2899 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:22.962862 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:23.439143 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:38:23.514803 2905 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:23.653204 1094 server/status/runtime.go:207 runtime stats: 351 MiB RSS, 105 goroutines, 16 MiB/12 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 35.56cgo/sec, 0.71/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:38:23.713329 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:38:23.822712 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:38:23.822939 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:38:23.868912 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.7 0.0 0.0 0.0 | |
| I160906 11:38:24.007453 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:24.134989 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:24.315686 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:24.363794 1223 server/status/runtime.go:207 runtime stats: 351 MiB RSS, 105 goroutines, 16 MiB/11 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.28cgo/sec, 0.69/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:38:24.561177 2826 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:24.562975 2826 gossip/client.go:189 [n3] node 3: sending [store:3 first-range system-db node:2 sentinel store:1 cluster-id store:2 node:3 node:1] | |
| I160906 11:38:24.563637 2826 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 store:2 system-db sentinel store:1 node:3 store:3 cluster-id] | |
| I160906 11:38:24.569048 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:24.804744 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:24.813277 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:24.813859 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:24.815792 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:24.870793 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:24.874118 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:24.928127 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:24.930552 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:24.931099 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:24.931266 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:25.126646 2934 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:25.965349 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:26.314938 2939 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:26.572907 2943 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:26.574780 2943 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:1 store:1 first-range store:2 node:2 sentinel node:3 store:3] | |
| I160906 11:38:26.575433 2943 gossip/client.go:189 [n3] node 3: sending [store:3 system-db node:2 sentinel store:1 store:2 cluster-id node:1 node:3 first-range] | |
| I160906 11:38:26.713272 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:27.007478 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:27.571742 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:27.717290 2960 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:27.878768 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:27.881958 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:27.882479 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:27.882654 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:28.574392 2912 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:28.576502 2912 gossip/client.go:189 [n3] node 3: sending [sentinel store:3 cluster-id store:2 node:2 store:1 node:3 first-range node:1 system-db] | |
| I160906 11:38:28.577169 2912 gossip/client.go:189 [n3] node 3: sending [cluster-id store:2 system-db store:3 first-range node:1 store:1 node:3 node:2 sentinel] | |
| I160906 11:38:28.715814 2991 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:28.967845 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:29.136527 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:29.317237 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:29.359382 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:29.713278 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:30.007393 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:30.574312 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:30.576075 2969 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:30.578271 2969 gossip/client.go:189 [n3] node 3: sending [store:1 cluster-id system-db first-range node:1 node:2 sentinel store:2 node:3 store:3] | |
| I160906 11:38:30.579146 2969 gossip/client.go:189 [n3] node 3: sending [store:3 cluster-id system-db first-range node:1 store:1 store:2 node:2 sentinel node:3] | |
| I160906 11:38:31.314969 3016 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:31.970283 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:32.138385 2999 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:32.362049 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:32.364992 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:32.365572 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:32.365788 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:32.548849 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:32.552430 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:32.577928 3028 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:32.580196 3028 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:2 cluster-id store:2 system-db store:1 node:1 sentinel] | |
| I160906 11:38:32.580823 3028 gossip/client.go:189 [n3] node 3: sending [node:2 store:2 node:3 first-range store:3 node:1 sentinel cluster-id system-db store:1] | |
| I160906 11:38:32.713353 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:32.817508 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:32.825634 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:32.826335 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:32.828768 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:33.007469 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:33.315405 3046 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:33.439480 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:38:33.576964 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:33.618493 1092 gossip/gossip.go:422 [n2] gossip status (ok, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 42/37 sent/received, bytes 17181B/13485B sent/received) | |
| W160906 11:38:33.618860 1092 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| I160906 11:38:33.643183 1094 server/status/runtime.go:207 runtime stats: 351 MiB RSS, 105 goroutines, 22 MiB/7.6 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.73cgo/sec, 0.66/0.00 %(u/s)time, 0.00 %gc (0x) | |
| W160906 11:38:33.822714 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:38:33.822918 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| W160906 11:38:33.845076 1130 storage/store.go:1034 [n2,s2] error gossiping system config: node unavailable; try another peer | |
| I160906 11:38:33.845848 1129 storage/replica.go:2845 [n2,s2,r1{/Min-/Table/11}] gossiping cluster id {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} from store 2, range 1 | |
| W160906 11:38:33.846098 1129 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| W160906 11:38:33.846189 1129 storage/store.go:1014 [n2,s2] error gossiping first range data: node unavailable; try another peer | |
| I160906 11:38:33.869664 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.8 0.0 0.0 0.0 | |
| I160906 11:38:34.138400 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:34.319033 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:34.358962 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 22 MiB/7.4 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.62cgo/sec, 0.67/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:38:34.359392 1221 gossip/gossip.go:422 [n3] gossip status (ok, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 972/32 sent/received, bytes 405455B/12946B sent/received) | |
| I160906 11:38:34.579632 3050 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:34.581382 3050 gossip/client.go:189 [n3] node 3: sending [node:2 sentinel node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:38:34.582000 3050 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 sentinel node:3 cluster-id system-db] | |
| I160906 11:38:34.972933 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:35.554953 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:35.557897 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:35.558428 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:35.558603 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:35.713291 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:36.007614 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:36.518652 3068 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:36.593966 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:36.603907 3073 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:36.609680 3073 gossip/client.go:189 [n3] node 3: sending [store:3 first-range system-db cluster-id node:1 node:2 node:3] | |
| I160906 11:38:36.621373 3073 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:38:37.707131 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:37.715313 3079 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:37.975451 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:38.602503 3100 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:38.604312 3100 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:38:38.604930 3100 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:38:38.713311 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:39.007397 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:39.139974 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:39.320579 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:39.603630 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:40.115391 3113 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:40.605937 3084 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:40.607351 3084 gossip/client.go:189 [n3] node 3: sending [system-db store:3 first-range node:1 node:2 node:3 cluster-id] | |
| I160906 11:38:40.607941 3084 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:38:40.709763 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:40.712308 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:40.712838 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:40.713045 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:40.978022 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:41.029614 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:41.074343 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:41.075018 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:41.077526 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:41.315095 3119 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:41.713321 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:42.007372 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:42.606201 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:42.623494 3154 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:42.625097 3154 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 cluster-id system-db first-range] | |
| I160906 11:38:42.625656 3154 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:38:42.714785 3165 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:43.211253 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:43.214244 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:43.214795 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:43.214966 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:43.439160 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:38:43.643163 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 14 MiB/13 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 30.10cgo/sec, 0.69/0.00 %(u/s)time, 0.00 %gc (1x) | |
| W160906 11:38:43.822718 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:38:43.822939 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:38:43.868896 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.5 0.0 0.0 0.0 | |
| I160906 11:38:43.980559 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:44.141462 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:44.322076 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:44.359437 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 14 MiB/13 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 30.20cgo/sec, 0.70/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:38:44.625132 3174 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:44.626811 3174 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 cluster-id system-db store:3 first-range] | |
| I160906 11:38:44.627392 3174 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:38:44.713391 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:44.914522 3186 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:45.007674 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:45.315408 3191 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:45.609265 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:45.858118 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:46.626759 3141 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:46.628447 3141 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:38:46.629063 3141 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:38:46.990440 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:47.115495 3202 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:47.714559 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:48.007397 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:48.611671 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:48.634716 3208 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:48.636243 3208 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:38:48.636904 3208 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:38:48.860854 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:48.863512 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:48.864054 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:48.864354 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:49.050946 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:49.059268 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:49.059949 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:49.062480 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:49.143267 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:49.323891 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:49.725618 3242 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:49.992987 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:50.314984 3220 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:50.637870 3224 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:50.639545 3224 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 cluster-id system-db store:3 first-range] | |
| I160906 11:38:50.640450 3224 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:38:50.713317 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:51.007428 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:51.315163 3248 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:51.614416 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:51.705442 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:51.708259 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:51.708808 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:51.708991 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:52.641972 3251 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:52.643584 3251 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 node:3 store:3 cluster-id] | |
| I160906 11:38:52.648338 3251 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:38:52.995452 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:53.439132 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:38:53.643389 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 19 MiB/9.3 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 33.40cgo/sec, 0.73/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:38:53.713395 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:38:53.824320 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| W160906 11:38:53.824436 1095 ts/db.go:104 node unavailable; try another peer | |
| I160906 11:38:53.869021 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:38:54.007660 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:54.144783 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:54.325436 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:54.358988 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 20 MiB/8.8 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 36.10cgo/sec, 0.71/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:38:54.619016 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:54.643765 3277 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:54.645359 3277 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:38:54.645982 3277 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:38:54.715432 3288 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:55.998008 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:56.114955 3297 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:56.328146 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:56.330721 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:56.331284 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:56.331503 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:56.435171 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:56.442520 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:56.443087 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:56.445172 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:56.507194 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:38:56.645296 3311 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:56.646688 3311 gossip/client.go:189 [n3] node 3: sending [system-db store:3 first-range node:1 node:2 node:3 cluster-id] | |
| I160906 11:38:56.647267 3311 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:38:56.713490 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:38:57.007415 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:38:57.621532 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:58.026322 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:38:58.315428 3326 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:58.650825 3262 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:38:58.652514 3262 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:38:58.653128 3262 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:38:59.000542 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:38:59.146253 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:59.326647 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:38:59.509551 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:38:59.512578 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:38:59.513111 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:59.513281 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:38:59.533149 3343 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:38:59.713304 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:00.007382 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:00.624125 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:00.652532 3349 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:00.654593 3349 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 node:3 store:3 cluster-id] | |
| I160906 11:39:00.655391 3349 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 cluster-id system-db store:3 first-range] | |
| I160906 11:39:00.876706 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:39:01.515665 3363 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:02.003032 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:02.315412 3370 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:02.654120 3374 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:02.655510 3374 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 cluster-id system-db store:3 first-range node:1] | |
| I160906 11:39:02.656383 3374 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 cluster-id system-db first-range node:1 node:2] | |
| I160906 11:39:02.713319 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:03.007428 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:03.314782 3385 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:03.439184 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:39:03.626654 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:03.643105 1094 server/status/runtime.go:207 runtime stats: 351 MiB RSS, 105 goroutines, 24 MiB/5.7 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 35.60cgo/sec, 0.76/0.00 %(u/s)time, 0.00 %gc (0x) | |
| W160906 11:39:03.826899 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:39:03.827030 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:39:03.869215 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.7 0.0 0.0 0.0 | |
| I160906 11:39:03.881330 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:03.888775 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:03.889334 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:03.891247 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:04.148201 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:04.328499 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:04.376897 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 24 MiB/5.3 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 32.74cgo/sec, 0.76/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:39:04.661439 3390 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:04.662863 3390 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 node:3 cluster-id system-db] | |
| I160906 11:39:04.663453 3390 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:39:04.672430 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:39:04.675797 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:39:04.684477 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:04.688849 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:04.690539 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:04.690787 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:05.005559 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:05.713368 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:06.007694 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:06.347283 3427 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:06.629177 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:06.674089 3401 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:06.675652 3401 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 first-range system-db cluster-id] | |
| I160906 11:39:06.676314 3401 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range system-db cluster-id node:1] | |
| I160906 11:39:06.914844 3432 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:07.314858 3437 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:07.682402 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:07.685226 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:07.685764 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:07.685930 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:07.715345 3463 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:08.008137 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:08.675684 3468 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:08.677570 3468 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:39:08.678165 3468 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 system-db cluster-id node:2 node:3] | |
| I160906 11:39:08.716495 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:09.007398 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:09.149583 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:09.330014 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:09.544055 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:39:09.634281 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:09.914883 3447 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:10.677317 3480 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:10.678715 3480 gossip/client.go:189 [n3] node 3: sending [first-range node:1 system-db cluster-id node:2 node:3 store:3] | |
| I160906 11:39:10.679367 3480 gossip/client.go:189 [n3] node 3: sending [first-range system-db cluster-id node:1 node:2 node:3 store:3] | |
| I160906 11:39:11.010585 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:11.315397 3492 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:11.713334 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:12.007389 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:12.555442 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:12.563134 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:12.563751 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:12.565749 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:12.636741 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:12.678929 3456 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:12.680595 3456 gossip/client.go:189 [n3] node 3: sending [store:3 cluster-id system-db first-range node:1 node:2 node:3] | |
| I160906 11:39:12.681205 3456 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 node:3 cluster-id system-db] | |
| I160906 11:39:12.715765 3516 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:13.348918 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:13.351350 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:13.351879 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:13.352046 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:13.439138 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:39:13.643310 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 16 MiB/11 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.40cgo/sec, 0.70/0.00 %(u/s)time, 0.00 %gc (1x) | |
| W160906 11:39:13.822721 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:39:13.822908 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:39:13.868894 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.8 0.0 0.0 0.0 | |
| I160906 11:39:14.014582 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:14.151125 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:14.331521 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:14.360884 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 17 MiB/11 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.55cgo/sec, 0.69/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:39:14.680727 3498 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:14.682159 3498 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:39:14.682786 3498 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:39:14.713364 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:14.914940 3541 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:14.982408 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:14.985326 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:14.985880 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:14.986050 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:15.007485 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:15.642388 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:16.683028 3532 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:16.684835 3532 gossip/client.go:189 [n3] node 3: sending [cluster-id node:2 node:3 store:3 first-range node:1 system-db] | |
| I160906 11:39:16.685461 3532 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 first-range system-db cluster-id] | |
| I160906 11:39:17.017043 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:17.713366 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:17.715387 3551 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:18.007671 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:18.051585 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:39:18.515396 3561 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:18.644986 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:18.684682 3567 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:18.686312 3567 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:39:18.686911 3567 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 first-range system-db cluster-id] | |
| I160906 11:39:19.156553 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:19.333403 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:20.019537 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:20.689134 3595 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:20.691787 3595 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:39:20.692608 3595 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:39:20.713270 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:20.715598 3607 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:21.007421 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:21.054335 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:21.056852 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:21.057345 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:21.057520 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:21.270478 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:21.277383 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:21.277928 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:21.279850 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:21.647800 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:22.690201 3574 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:22.691602 3574 gossip/client.go:189 [n3] node 3: sending [node:1 system-db cluster-id node:2 node:3 store:3 first-range] | |
| I160906 11:39:22.692290 3574 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:39:22.915056 3585 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:23.021971 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:23.439177 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:39:23.515043 3641 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:23.643111 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 22 MiB/7.4 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 32.90cgo/sec, 0.87/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:39:23.713313 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:23.734321 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:23.739661 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:23.740966 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:23.741556 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| W160906 11:39:23.822732 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:39:23.822969 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:39:23.868891 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:39:23.925848 3647 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:24.010903 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:24.158053 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:24.334898 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:24.358964 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 22 MiB/6.7 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 37.01cgo/sec, 0.89/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:39:24.650295 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:24.691739 3623 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:24.693250 3623 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 node:3 cluster-id system-db] | |
| I160906 11:39:24.693827 3623 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:39:25.915457 3667 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:26.024811 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:26.693459 3653 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:26.695147 3653 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:39:26.695737 3653 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:39:26.713313 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:27.007432 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:27.515375 3675 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:27.652834 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:28.695051 3679 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:28.696947 3679 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:39:28.697583 3679 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:39:29.027166 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:29.153053 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:29.155646 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:29.156301 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:29.156498 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:29.164169 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:29.314985 3690 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:29.345347 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:29.471083 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:29.478209 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:29.478741 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:29.480739 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:29.713301 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:30.007381 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:30.655350 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:30.696664 3707 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:30.698058 3707 gossip/client.go:189 [n3] node 3: sending [first-range system-db cluster-id node:1 node:2 node:3 store:3] | |
| I160906 11:39:30.698640 3707 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 system-db cluster-id node:2 node:3] | |
| I160906 11:39:30.915049 3718 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:31.083390 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:31.086736 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:31.087398 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:31.087622 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:31.315003 3727 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:32.037451 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:32.698363 3747 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:32.699638 3747 gossip/client.go:189 [n3] node 3: sending [store:3 first-range system-db cluster-id node:1 node:2 node:3] | |
| I160906 11:39:32.701738 3747 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range node:1] | |
| I160906 11:39:32.713240 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:32.715468 3759 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:33.007400 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:33.439179 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:39:33.514944 3734 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:33.618510 1092 gossip/gossip.go:422 [n2] gossip status (stalled, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 42/37 sent/received, bytes 17181B/13485B sent/received) | |
| W160906 11:39:33.619018 1092 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| I160906 11:39:33.642988 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 15 MiB/12 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 38.50cgo/sec, 0.95/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:39:33.658031 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| W160906 11:39:33.822733 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:39:33.822961 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| W160906 11:39:33.842829 1130 storage/store.go:1034 [n2,s2] error gossiping system config: node unavailable; try another peer | |
| I160906 11:39:33.845904 1129 storage/replica.go:2845 [n2,s2,r1{/Min-/Table/11}] gossiping cluster id {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} from store 2, range 1 | |
| W160906 11:39:33.846170 1129 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| W160906 11:39:33.846273 1129 storage/store.go:1014 [n2,s2] error gossiping first range data: node unavailable; try another peer | |
| I160906 11:39:33.868944 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.8 0.0 0.0 0.0 | |
| I160906 11:39:34.165930 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:34.347141 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:34.359069 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 15 MiB/12 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 37.60cgo/sec, 0.95/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:39:34.359468 1221 gossip/gossip.go:422 [n3] gossip status (stalled, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 1394/32 sent/received, bytes 637513B/12946B sent/received) | |
| I160906 11:39:34.699992 3743 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:34.701728 3743 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:39:34.702335 3743 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:1 node:2 node:3 store:3 first-range] | |
| I160906 11:39:35.039994 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:35.713350 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:36.007404 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:36.660763 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:36.701771 3788 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:36.703138 3788 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:39:36.703740 3788 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:39:37.072533 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:37.075170 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:37.075715 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:37.075905 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:37.115025 3804 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:37.322400 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:37.329010 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:37.329516 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:37.331395 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:38.042445 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:38.703538 3764 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:38.705202 3764 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:39:38.705803 3764 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range node:1] | |
| I160906 11:39:38.716574 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:39.007637 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:39.115357 3815 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:39.167413 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:39.348640 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:39.629348 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:39.632353 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:39.632897 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:39.633068 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:39.663137 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:40.705078 3825 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:40.706669 3825 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:39:40.707239 3825 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:39:41.044934 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:41.340725 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:39:41.515285 3839 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:41.713479 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:42.007404 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:42.665598 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:42.706641 3846 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:42.708062 3846 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range system-db cluster-id node:1] | |
| I160906 11:39:42.708732 3846 gossip/client.go:189 [n3] node 3: sending [store:3 cluster-id system-db first-range node:1 node:2 node:3] | |
| I160906 11:39:42.715237 3857 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:43.314676 3862 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:43.447447 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:39:43.643069 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 19 MiB/8.9 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 33.40cgo/sec, 0.98/0.00 %(u/s)time, 0.00 %gc (0x) | |
| W160906 11:39:43.822726 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:39:43.822962 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:39:43.868883 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:39:44.047325 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:44.168918 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:44.343009 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:44.345557 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:44.346066 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:44.346239 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:44.350193 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:44.359266 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 20 MiB/8.6 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.60cgo/sec, 0.98/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:39:44.708189 3775 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:44.709581 3775 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:39:44.710164 3775 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:39:44.713225 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:44.803800 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:44.810539 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:44.811059 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:44.812806 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:45.007604 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:45.514845 3887 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:45.668479 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:46.515201 3895 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:46.714589 3899 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:46.716256 3899 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:39:46.716842 3899 gossip/client.go:189 [n3] node 3: sending [node:3 cluster-id system-db store:3 first-range node:1 node:2] | |
| I160906 11:39:47.024353 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:47.027311 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:47.027949 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:47.028166 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:47.050188 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:47.713279 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:48.007369 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:48.671415 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:48.716224 3918 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:48.717976 3918 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:39:48.718553 3918 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:39:49.170743 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:49.351977 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:50.052648 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:50.115391 3932 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:50.713283 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:50.717608 3938 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:50.719084 3938 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:39:50.719667 3938 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:39:51.007348 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:51.674013 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:51.959759 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:51.962443 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:51.963020 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:51.963195 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:52.135138 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:52.142249 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:52.142789 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:52.155205 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:52.719269 3959 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:52.720956 3959 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range node:1] | |
| I160906 11:39:52.721561 3959 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 cluster-id system-db first-range node:1 node:2] | |
| I160906 11:39:53.055561 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:53.439175 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:39:53.643228 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 24 MiB/5.1 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.30cgo/sec, 0.99/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:39:53.713334 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:39:53.823079 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:39:53.823198 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:39:53.868929 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.4 0.0 0.0 0.0 | |
| I160906 11:39:54.007772 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:54.172288 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:54.313106 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:39:54.316060 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:39:54.316640 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:54.316816 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:39:54.353376 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:54.359324 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 25 MiB/4.7 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.30cgo/sec, 0.99/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:39:54.515147 3977 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:54.676685 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:54.720882 3986 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:54.722427 3986 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:39:54.722990 3986 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 cluster-id system-db first-range node:1 node:2] | |
| I160906 11:39:55.515484 3982 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:56.058085 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:56.713282 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:39:56.722396 4005 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:56.723983 4005 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:39:56.724609 4005 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 cluster-id system-db first-range] | |
| I160906 11:39:57.007363 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:39:57.268318 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:39:57.686530 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:58.724693 4020 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:39:58.726412 4020 gossip/client.go:189 [n3] node 3: sending [first-range system-db cluster-id node:1 node:2 node:3 store:3] | |
| I160906 11:39:58.727001 4020 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:39:59.060556 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:39:59.115152 4032 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:39:59.173712 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:59.354784 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:39:59.713299 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:00.007403 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:00.271123 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:00.273702 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:00.274226 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:00.274389 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:00.551201 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:00.557960 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:00.558477 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:00.560226 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:00.688982 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:00.726241 4047 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:00.727855 4047 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range node:1] | |
| I160906 11:40:00.728444 4047 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:40:02.063030 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:02.713290 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:02.727764 3998 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:02.729375 3998 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:40:02.729948 3998 gossip/client.go:189 [n3] node 3: sending [node:1 system-db cluster-id node:2 node:3 store:3 first-range] | |
| I160906 11:40:02.856317 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:02.858980 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:02.859481 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:02.859646 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:03.007385 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:03.114892 4078 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:03.439144 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:40:03.644422 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 16 MiB/11 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 37.10cgo/sec, 0.97/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:40:03.691485 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| W160906 11:40:03.822701 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:40:03.822912 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:40:03.868888 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:40:04.175450 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:04.356574 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:04.359207 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 17 MiB/11 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 38.50cgo/sec, 0.97/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:40:04.737819 4059 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:04.739544 4059 gossip/client.go:189 [n3] node 3: sending [store:3 first-range system-db cluster-id node:1 node:2 node:3] | |
| I160906 11:40:04.740163 4059 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:40:04.826786 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:05.065457 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:05.713281 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:06.007418 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:06.693934 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:06.739303 4106 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:06.740959 4106 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:40:06.741535 4106 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:40:06.916244 4118 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:07.829826 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:07.836669 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:07.837198 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:07.838904 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:07.915176 4087 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:08.067834 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:08.115306 4093 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:08.713283 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:08.740909 4097 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:08.742575 4097 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:40:08.743156 4097 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:40:08.899968 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:08.902469 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:08.902958 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:08.903121 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:09.012679 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:09.114694 4144 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:09.176891 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:09.357989 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:09.696449 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:10.742472 4127 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:10.743848 4127 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:40:10.744538 4127 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:40:10.760430 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:10.763233 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:10.763740 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:10.763917 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:10.871593 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:40:11.070558 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:11.713284 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:12.007865 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:12.698819 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:12.715221 4178 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:12.744077 4183 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:12.745779 4183 gossip/client.go:189 [n3] node 3: sending [store:3 cluster-id system-db first-range node:1 node:2 node:3] | |
| I160906 11:40:12.746334 4183 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 cluster-id system-db store:3 first-range node:1] | |
| I160906 11:40:13.272226 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:13.275495 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:13.439138 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:40:13.643420 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 21 MiB/7.9 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.40cgo/sec, 0.98/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:40:13.714998 4194 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| W160906 11:40:13.822719 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:40:13.822945 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:40:13.868878 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.7 0.0 0.0 0.0 | |
| I160906 11:40:14.072977 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:14.178332 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:14.359384 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:14.364570 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 21 MiB/7.6 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.38cgo/sec, 0.98/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:40:14.713307 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:14.765204 4154 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:14.768678 4154 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:40:14.770076 4154 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:40:15.007418 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:15.701282 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:16.278439 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:16.285667 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:16.286214 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:16.288067 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:16.315106 4226 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:16.754538 4230 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:16.755974 4230 gossip/client.go:189 [n3] node 3: sending [cluster-id node:1 node:2 node:3 store:3 first-range system-db] | |
| I160906 11:40:16.756652 4230 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:40:16.995202 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:16.997872 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:16.998447 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:16.998615 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:17.075567 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:17.713294 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:18.007410 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:18.518653 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:18.521639 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:18.522141 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:18.522305 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:18.703830 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:18.715053 4254 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:18.756229 4258 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:18.757729 4258 gossip/client.go:189 [n3] node 3: sending [store:3 first-range system-db cluster-id node:1 node:2 node:3] | |
| I160906 11:40:18.758337 4258 gossip/client.go:189 [n3] node 3: sending [node:1 system-db cluster-id node:2 node:3 store:3 first-range] | |
| I160906 11:40:19.180265 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:19.361306 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:20.078190 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:20.515303 4213 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:20.728476 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:20.758394 4217 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:20.760170 4217 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:40:20.760749 4217 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 first-range system-db cluster-id] | |
| I160906 11:40:21.007424 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:21.129245 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:21.133079 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:21.706392 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:21.715444 4283 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:22.114725 4270 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:22.760039 4289 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:22.762001 4289 gossip/client.go:189 [n3] node 3: sending [system-db node:3 store:3 first-range node:1 node:2 cluster-id] | |
| I160906 11:40:22.762579 4289 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:40:23.080769 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:23.439225 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:40:23.643179 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 26 MiB/4.2 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.70cgo/sec, 0.98/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:40:23.713318 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:40:23.822740 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:40:23.822970 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:40:23.872027 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.5 0.0 0.0 0.0 | |
| I160906 11:40:24.007387 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:24.115296 4300 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:24.136366 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:24.143699 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:24.144364 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:24.146325 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:24.202261 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:24.359054 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 14 MiB/13 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.62cgo/sec, 0.98/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:40:24.363706 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:24.486381 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:24.488954 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:24.489455 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:24.489621 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:24.709167 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:24.761768 4330 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:24.763168 4330 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:40:24.763757 4330 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:40:25.714882 4309 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:26.083195 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:26.713319 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:26.763356 4315 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:26.764856 4315 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:40:26.765458 4315 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 cluster-id system-db first-range node:1 node:2] | |
| I160906 11:40:26.914894 4359 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:27.007636 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:27.068385 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:27.071238 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:27.071786 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:27.071980 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:27.115746 4346 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:27.711643 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:28.765052 4363 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:28.766687 4363 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:40:28.767219 4363 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:40:28.921650 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:28.926040 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:29.085854 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:29.115416 4380 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:29.205564 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:29.365213 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:29.713298 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:30.007394 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:30.714282 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:30.766679 4353 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:30.768373 4353 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:40:30.768986 4353 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:40:31.914983 4398 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:31.931091 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:31.938460 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:31.939028 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:31.941029 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:32.088370 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:32.551293 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:32.553963 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:32.554497 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:32.556755 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:32.713543 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:32.715418 4422 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:32.768373 4426 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:32.770036 4426 gossip/client.go:189 [n3] node 3: sending [store:3 cluster-id system-db first-range node:1 node:2 node:3] | |
| I160906 11:40:32.770639 4426 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:40:33.007412 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:33.439166 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:40:33.618106 1092 gossip/gossip.go:422 [n2] gossip status (stalled, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 42/37 sent/received, bytes 17181B/13485B sent/received) | |
| W160906 11:40:33.618497 1092 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| I160906 11:40:33.643218 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 19 MiB/9.4 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 36.10cgo/sec, 0.97/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:40:33.716749 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| W160906 11:40:33.822717 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:40:33.822944 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| W160906 11:40:33.842723 1130 storage/store.go:1034 [n2,s2] error gossiping system config: node unavailable; try another peer | |
| I160906 11:40:33.845757 1129 storage/replica.go:2845 [n2,s2,r1{/Min-/Table/11}] gossiping cluster id {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} from store 2, range 1 | |
| W160906 11:40:33.846006 1129 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| W160906 11:40:33.846091 1129 storage/store.go:1014 [n2,s2] error gossiping first range data: node unavailable; try another peer | |
| I160906 11:40:33.868890 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.9 0.0 0.0 0.0 | |
| I160906 11:40:34.207403 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:34.358512 1221 gossip/gossip.go:422 [n3] gossip status (stalled, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 1814/32 sent/received, bytes 869413B/12946B sent/received) | |
| I160906 11:40:34.360903 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 19 MiB/9.3 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 33.39cgo/sec, 0.97/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:40:34.367014 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:34.684463 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:34.687308 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:34.687858 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:34.688039 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:34.769897 4446 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:34.771414 4446 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 node:3 cluster-id system-db] | |
| I160906 11:40:34.772014 4446 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:40:35.090784 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:35.713316 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:36.007397 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:36.515391 4459 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:36.719210 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:36.771561 4408 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:36.773382 4408 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:40:36.774007 4408 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 first-range system-db cluster-id] | |
| I160906 11:40:36.882223 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:37.315497 4470 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:38.093408 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:38.515338 4483 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:38.713307 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:38.773137 4487 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:38.774499 4487 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:40:38.775080 4487 gossip/client.go:189 [n3] node 3: sending [node:1 system-db cluster-id node:2 node:3 store:3 first-range] | |
| I160906 11:40:39.007385 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:39.208964 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:39.368564 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:39.721772 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:39.884740 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:39.887243 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:39.887777 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:39.887958 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:40.013453 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:40.021757 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:40.022456 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:40.024873 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:40.221640 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:40.225307 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:40.774754 4512 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:40.776286 4512 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range system-db cluster-id node:1 node:2] | |
| I160906 11:40:40.776907 4512 gossip/client.go:189 [n3] node 3: sending [node:3 cluster-id system-db store:3 first-range node:1 node:2] | |
| I160906 11:40:40.915034 4523 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:41.095885 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:41.716510 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:42.007411 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:42.518379 4531 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:42.724289 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:42.777143 4536 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:42.778806 4536 gossip/client.go:189 [n3] node 3: sending [system-db node:3 store:3 first-range node:1 node:2 cluster-id] | |
| I160906 11:40:42.779413 4536 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:40:43.227847 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:43.230747 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:43.231280 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:43.231461 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:43.314988 4551 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:43.439157 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:40:43.643110 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 23 MiB/6.0 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.00cgo/sec, 0.97/0.00 %(u/s)time, 0.00 %gc (0x) | |
| W160906 11:40:43.822716 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:40:43.822936 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:40:43.868893 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:40:44.098270 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:44.210431 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:44.359277 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 23 MiB/5.9 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.51cgo/sec, 0.97/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:40:44.370071 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:44.713309 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:44.778702 4556 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:44.780797 4556 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:40:44.781568 4556 gossip/client.go:189 [n3] node 3: sending [system-db node:3 store:3 first-range node:1 node:2 cluster-id] | |
| I160906 11:40:45.007632 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:45.726799 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:46.315387 4567 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:46.780347 4479 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:46.782401 4479 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:40:46.783128 4479 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:40:47.100758 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:47.293431 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:47.295972 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:47.296539 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:47.296720 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:47.483771 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:47.491122 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:47.491677 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:47.493575 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:47.515774 4598 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:47.713581 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:48.007410 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:48.276343 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:48.729290 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:48.781465 4587 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:48.782461 4587 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 system-db cluster-id node:2] | |
| I160906 11:40:48.782913 4587 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:40:49.212354 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:49.371979 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:50.103775 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:50.115773 4607 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:50.713396 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:50.783771 4627 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:50.785472 4627 gossip/client.go:189 [n3] node 3: sending [first-range node:1 system-db cluster-id node:2 node:3 store:3] | |
| I160906 11:40:50.786095 4627 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:40:51.007431 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:51.280405 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:51.284493 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:51.285243 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:51.285495 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:51.515103 4640 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:51.731996 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:51.796321 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:52.715446 4650 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:52.785491 4654 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:52.787070 4654 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 cluster-id system-db store:3 first-range node:1] | |
| I160906 11:40:52.787637 4654 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:40:53.106546 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:53.439230 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:40:53.643305 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 15 MiB/12 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.50cgo/sec, 0.96/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:40:53.713328 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:40:53.822744 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:40:53.822972 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:40:53.872157 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.5 0.0 0.0 0.0 | |
| I160906 11:40:54.007443 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:54.213969 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:54.359014 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 16 MiB/12 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 33.70cgo/sec, 0.96/0.01 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:40:54.373606 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:54.514969 4667 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:54.734774 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:54.787157 4673 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:54.789104 4673 gossip/client.go:189 [n3] node 3: sending [node:3 cluster-id system-db store:3 first-range node:1 node:2] | |
| I160906 11:40:54.789662 4673 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:40:54.798805 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:54.801481 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:54.802036 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:54.802217 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:55.166861 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:55.175394 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:55.176082 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:55.178661 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:55.715483 4693 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:56.109329 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:56.459204 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:56.525918 4703 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:56.713316 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:56.789005 4619 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:56.790801 4619 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 node:3 store:3 cluster-id] | |
| I160906 11:40:56.791387 4619 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:40:57.007461 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:40:57.737600 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:58.115240 4707 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:40:58.809846 4712 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:40:58.813822 4712 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range system-db cluster-id node:1] | |
| I160906 11:40:58.814447 4712 gossip/client.go:189 [n3] node 3: sending [first-range system-db cluster-id node:1 node:2 node:3 store:3] | |
| I160906 11:40:59.112062 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:40:59.215550 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:59.375129 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:40:59.462892 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:40:59.465942 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:40:59.466560 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:59.466746 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:40:59.621212 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:40:59.713578 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:40:59.715873 4737 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:00.007430 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:00.715509 4743 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:00.740345 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:00.811417 4748 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:00.813290 4748 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 cluster-id system-db store:3 first-range] | |
| I160906 11:41:00.814008 4748 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 system-db cluster-id node:2 node:3] | |
| I160906 11:41:01.915043 4772 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:02.114842 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:02.624828 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:02.631826 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:02.632458 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:02.634181 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:02.721574 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:02.793606 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:02.796390 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:02.796934 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:02.797111 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:02.813139 4785 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:02.814932 4785 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range node:1] | |
| I160906 11:41:02.815519 4785 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 cluster-id system-db first-range] | |
| I160906 11:41:03.007440 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:03.315810 4797 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:03.439201 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:41:03.643242 1094 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 21 MiB/7.7 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 41.20cgo/sec, 0.96/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:41:03.743052 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| W160906 11:41:03.822733 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:41:03.822961 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:41:03.868918 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.9 0.0 0.0 0.0 | |
| I160906 11:41:04.219771 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:04.359386 1223 server/status/runtime.go:207 runtime stats: 352 MiB RSS, 105 goroutines, 22 MiB/7.1 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 44.10cgo/sec, 0.96/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:41:04.377239 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:04.814795 4802 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:04.816686 4802 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 cluster-id system-db first-range node:1 node:2] | |
| I160906 11:41:04.817308 4802 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:41:05.086484 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:41:05.117668 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:05.713295 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:05.914990 4814 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:06.007325 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:06.745846 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:06.816398 4821 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:06.818104 4821 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 cluster-id system-db store:3 first-range node:1] | |
| I160906 11:41:06.818831 4821 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 cluster-id system-db first-range node:1 node:2] | |
| I160906 11:41:06.915539 4832 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:07.515531 4765 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:08.088923 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:08.092535 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:08.093221 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:08.093449 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:08.119566 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:08.713353 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:08.817833 4855 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:08.819524 4855 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:41:08.820131 4855 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:41:09.007438 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:09.221310 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:09.378843 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:09.748406 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:09.915514 4869 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:10.074413 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:10.083160 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:10.083883 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:10.085770 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:10.515484 4838 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:10.819738 4842 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:10.821662 4842 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:41:10.822297 4842 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:41:11.043090 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:11.045951 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:11.046521 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:11.046708 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:11.122350 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:11.532024 4901 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:11.749183 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:12.007410 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:12.750477 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:12.830361 4887 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:12.833394 4887 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:41:12.834156 4887 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:41:13.115101 4914 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:13.353009 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:41:13.439188 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:41:13.643236 1094 server/status/runtime.go:207 runtime stats: 355 MiB RSS, 105 goroutines, 27 MiB/4.1 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 42.60cgo/sec, 0.98/0.00 %(u/s)time, 0.00 %gc (0x) | |
| W160906 11:41:13.822731 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:41:13.822943 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:41:13.870200 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.8 0.0 0.0 0.0 | |
| I160906 11:41:14.125094 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:14.222878 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:14.379870 1223 server/status/runtime.go:207 runtime stats: 357 MiB RSS, 105 goroutines, 27 MiB/3.7 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 39.72cgo/sec, 0.97/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:41:14.385336 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:14.713306 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:14.832293 4921 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:14.833798 4921 gossip/client.go:189 [n3] node 3: sending [store:3 first-range system-db cluster-id node:1 node:2 node:3] | |
| I160906 11:41:14.834430 4921 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:41:15.007428 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:15.753158 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:15.915485 4910 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:16.115547 4947 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:16.355420 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:16.358419 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:16.358957 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:16.359153 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:16.833856 4936 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:16.835676 4936 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:41:16.836301 4936 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:41:17.127689 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:17.713310 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:18.007377 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:18.264046 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:18.272920 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:18.273574 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:18.275445 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:18.755816 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:18.835454 4966 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:18.837139 4966 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:41:18.837715 4966 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:41:18.915010 4956 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:18.977193 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:18.980149 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:18.980834 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:18.981086 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:19.224750 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:19.387173 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:20.130211 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:20.713358 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:20.837234 4980 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:20.839044 4980 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:41:20.839787 4980 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 node:3 store:3 cluster-id] | |
| I160906 11:41:20.915157 4997 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:21.007461 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:21.515052 5002 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:21.758510 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:22.515507 5009 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:22.838780 5013 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:22.841083 5013 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 system-db cluster-id node:2] | |
| I160906 11:41:22.841738 5013 gossip/client.go:189 [n3] node 3: sending [cluster-id node:2 node:3 store:3 first-range node:1 system-db] | |
| I160906 11:41:23.132727 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:23.439185 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:41:23.643158 1094 server/status/runtime.go:207 runtime stats: 356 MiB RSS, 105 goroutines, 18 MiB/11 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 30.50cgo/sec, 1.01/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:41:23.713302 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:41:23.822736 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:41:23.822957 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:41:23.870043 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:41:24.007408 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:24.151281 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:24.154279 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:24.154845 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:24.155041 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:24.226663 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:24.358997 1223 server/status/runtime.go:207 runtime stats: 356 MiB RSS, 105 goroutines, 19 MiB/10 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 29.06cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:41:24.388788 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:24.682924 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:41:24.761578 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:24.840825 5032 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:24.842426 5032 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:41:24.843056 5032 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:41:25.739924 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:25.748770 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:25.749419 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:25.751359 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:26.115070 5058 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:26.135288 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:26.713547 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:26.715305 5048 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:26.842373 5063 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:26.843797 5063 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range system-db cluster-id node:1 node:2] | |
| I160906 11:41:26.844411 5063 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 node:3 store:3 cluster-id] | |
| I160906 11:41:27.007430 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:27.685833 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:27.689067 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:27.689763 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:27.689991 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:27.764126 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:28.115435 5054 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:28.844293 5090 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:28.845847 5090 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:41:28.846451 5090 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:41:29.138081 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:29.228281 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:29.390396 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:29.713305 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:30.007634 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:30.515015 5103 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:30.766799 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:30.845875 5081 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:30.847421 5081 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:41:30.848015 5081 gossip/client.go:189 [n3] node 3: sending [system-db node:3 store:3 first-range node:1 node:2 cluster-id] | |
| I160906 11:41:31.514788 5124 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:32.140626 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:32.320773 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:32.324330 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:32.325013 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:32.325241 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:32.713271 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:32.847505 5132 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:32.849225 5132 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:41:32.849828 5132 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 cluster-id system-db store:3 first-range node:1] | |
| I160906 11:41:33.007444 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:33.439167 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| W160906 11:41:33.578848 1035 storage/store_pool.go:83 store 1 on node 1 is now considered offline | |
| I160906 11:41:33.618452 1092 gossip/gossip.go:422 [n2] gossip status (stalled, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 42/37 sent/received, bytes 17181B/13485B sent/received) | |
| W160906 11:41:33.618820 1092 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| I160906 11:41:33.644375 1094 server/status/runtime.go:207 runtime stats: 356 MiB RSS, 105 goroutines, 23 MiB/7.0 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 33.80cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:41:33.773117 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| W160906 11:41:33.822734 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:41:33.822945 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| W160906 11:41:33.842014 1035 storage/store_pool.go:83 store 2 on node 2 is now considered offline | |
| W160906 11:41:33.842776 1130 storage/store.go:1034 [n2,s2] error gossiping system config: node unavailable; try another peer | |
| I160906 11:41:33.845801 1129 storage/replica.go:2845 [n2,s2,r1{/Min-/Table/11}] gossiping cluster id {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} from store 2, range 1 | |
| W160906 11:41:33.845990 1129 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| W160906 11:41:33.846082 1129 storage/store.go:1014 [n2,s2] error gossiping first range data: node unavailable; try another peer | |
| I160906 11:41:33.868852 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.5 0.0 0.0 0.0 | |
| I160906 11:41:34.115043 5114 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:34.230497 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| W160906 11:41:34.244369 1113 storage/store_pool.go:83 store 1 on node 1 is now considered offline | |
| I160906 11:41:34.358922 1221 gossip/gossip.go:422 [n3] gossip status (stalled, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 2234/32 sent/received, bytes 1101313B/12946B sent/received) | |
| I160906 11:41:34.360031 1223 server/status/runtime.go:207 runtime stats: 356 MiB RSS, 106 goroutines, 24 MiB/6.8 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 35.30cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:41:34.387765 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:34.393180 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:34.397922 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:34.398643 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:34.400841 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:34.848971 5151 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:34.851039 5151 gossip/client.go:189 [n3] node 3: sending [first-range system-db cluster-id node:1 node:2 node:3 store:3] | |
| I160906 11:41:34.851756 5151 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:41:35.027754 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:35.030375 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:35.030932 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:35.031124 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:35.115133 5167 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:35.143081 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:35.713318 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:36.007398 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:36.775738 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:36.850577 5175 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:36.852352 5175 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:41:36.852985 5175 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:41:38.114958 5188 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:38.145642 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:38.313350 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:38.713291 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:38.852145 5193 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:38.853525 5193 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 node:3 cluster-id system-db] | |
| I160906 11:41:38.854115 5193 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:41:39.007438 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:39.232019 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:39.394684 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:39.514716 5119 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:39.778313 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:40.752920 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:40.756484 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:40.757162 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:40.757384 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:40.853813 5210 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:40.855489 5210 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:41:40.856730 5210 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:41:41.148304 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:41.313346 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:41.714117 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:41.716184 5221 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:42.007422 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:42.251077 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:42.259724 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:42.260381 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:42.262245 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:42.780890 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:42.855427 5230 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:42.857160 5230 gossip/client.go:189 [n3] node 3: sending [system-db node:3 store:3 first-range node:1 node:2 cluster-id] | |
| I160906 11:41:42.857745 5230 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range system-db cluster-id node:1 node:2] | |
| I160906 11:41:43.114882 5258 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:43.116871 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:43.121010 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:43.121877 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:43.122133 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:43.439481 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:41:43.643116 1094 server/status/runtime.go:207 runtime stats: 357 MiB RSS, 106 goroutines, 16 MiB/13 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.70cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| W160906 11:41:43.822713 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:41:43.822940 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:41:43.868889 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:41:44.150818 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:44.233575 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:44.313319 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:44.359010 1223 server/status/runtime.go:207 runtime stats: 357 MiB RSS, 106 goroutines, 16 MiB/13 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.80cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:41:44.396281 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:44.713301 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:44.857177 5266 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:44.858968 5266 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:41:44.859726 5266 gossip/client.go:189 [n3] node 3: sending [store:3 cluster-id system-db first-range node:1 node:2 node:3] | |
| I160906 11:41:45.007828 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:45.115005 5276 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:45.783346 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:46.368226 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:41:46.372322 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:41:46.858707 5246 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:46.860785 5246 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:41:46.861532 5246 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:41:46.914930 5306 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:47.153285 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:47.313300 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:47.713327 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:48.007383 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:48.714922 5312 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:48.785801 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:48.860255 5317 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:48.861616 5317 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:41:48.862210 5317 gossip/client.go:189 [n3] node 3: sending [first-range system-db cluster-id node:1 node:2 node:3 store:3] | |
| I160906 11:41:49.235413 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:49.374998 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:49.378463 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:49.379114 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:49.379331 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:49.397556 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:50.155753 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:50.246750 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:50.254490 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:50.255163 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:50.257395 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:50.313290 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:50.514920 5333 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:50.713320 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:50.861824 5293 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:50.863255 5293 gossip/client.go:189 [n3] node 3: sending [first-range system-db cluster-id node:1 node:2 node:3 store:3] | |
| I160906 11:41:50.863853 5293 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 node:3 store:3 cluster-id] | |
| I160906 11:41:51.007423 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:51.067803 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:51.070399 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:51.070941 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:51.071127 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:51.114710 5358 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:51.788253 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:52.863470 5364 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:52.865221 5364 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:41:52.865985 5364 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:41:53.158203 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:53.313299 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:53.439161 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:41:53.643045 1094 server/status/runtime.go:207 runtime stats: 357 MiB RSS, 106 goroutines, 20 MiB/9.2 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 33.30cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:41:53.713299 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:41:53.822708 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:41:53.822913 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:41:53.870033 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.5 0.0 0.0 0.0 | |
| I160906 11:41:53.883475 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:41:53.886783 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:41:54.008046 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:54.236971 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:54.358926 1223 server/status/runtime.go:207 runtime stats: 357 MiB RSS, 106 goroutines, 21 MiB/8.8 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 37.00cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:41:54.399087 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:54.514942 5381 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:54.790819 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:54.865271 5339 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:54.866678 5339 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 node:3 cluster-id system-db] | |
| I160906 11:41:54.867268 5339 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:41:55.515361 5398 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:56.115327 5387 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:56.160683 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| W160906 11:41:56.285581 1035 storage/store_pool.go:83 store 3 on node 3 is now considered offline | |
| I160906 11:41:56.313319 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| W160906 11:41:56.328166 1113 storage/store_pool.go:83 store 2 on node 2 is now considered offline | |
| I160906 11:41:56.713354 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:41:56.866889 5392 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:56.868313 5392 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:41:56.868930 5392 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:41:56.888941 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:56.892130 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:56.892784 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:56.892999 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:57.007445 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:57.793286 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:58.553556 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:58.560671 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:58.561228 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:58.563049 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:58.868448 5408 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:41:58.870197 5408 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:41:58.870784 5408 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:1 node:2 node:3 store:3 first-range] | |
| I160906 11:41:59.115356 5452 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:41:59.163090 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:41:59.238531 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:59.280034 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:41:59.282934 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:41:59.283597 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:59.283842 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:41:59.313621 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:41:59.400654 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:41:59.713273 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:00.007400 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:00.795781 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:00.870108 5463 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:00.871512 5463 gossip/client.go:189 [n3] node 3: sending [system-db node:3 store:3 first-range node:1 node:2 cluster-id] | |
| I160906 11:42:00.872136 5463 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:42:01.223910 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:42:01.915136 5432 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:02.165773 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:02.313397 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:02.515596 5476 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:02.713383 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:02.871749 5480 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:02.873581 5480 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range node:1] | |
| I160906 11:42:02.874302 5480 gossip/client.go:189 [n3] node 3: sending [store:3 first-range system-db cluster-id node:1 node:2 node:3] | |
| I160906 11:42:03.007406 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:03.439174 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:42:03.643045 1094 server/status/runtime.go:207 runtime stats: 357 MiB RSS, 106 goroutines, 26 MiB/4.8 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 38.40cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:42:03.798456 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| W160906 11:42:03.822723 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:42:03.822928 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:42:03.868910 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:42:04.226499 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:04.230368 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:04.230999 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:04.231253 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:04.240517 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:04.359357 1223 server/status/runtime.go:207 runtime stats: 357 MiB RSS, 106 goroutines, 26 MiB/4.7 MiB/41 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.80cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:42:04.402535 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:04.873356 5440 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:04.875090 5440 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:42:04.875736 5440 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:42:04.915421 5495 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:05.168303 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:05.317530 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:05.713294 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:05.716252 5515 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:06.007417 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:06.175725 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:06.184222 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:06.184848 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:06.186734 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:06.801354 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:06.877082 5505 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:06.878818 5505 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:42:06.879420 5505 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 node:3 store:3 cluster-id] | |
| I160906 11:42:07.651249 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:07.654414 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:07.655128 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:07.655348 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:08.170831 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:08.313370 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:08.713304 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:08.878617 5521 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:08.880296 5521 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:42:08.880883 5521 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:42:08.915421 5548 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:09.007604 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:09.114754 5553 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:09.241978 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:09.404061 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:09.765059 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:42:09.769152 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:42:09.803769 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:10.315334 5571 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:10.880357 5575 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:10.881776 5575 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:42:10.882383 5575 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:1 node:2 node:3 store:3 first-range] | |
| I160906 11:42:11.173365 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:11.313308 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:11.713320 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:12.007386 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:12.771629 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:12.775066 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:12.775677 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:12.775894 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:12.806174 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:12.881903 5592 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:12.883394 5592 gossip/client.go:189 [n3] node 3: sending [first-range node:1 system-db cluster-id node:2 node:3 store:3] | |
| I160906 11:42:12.883970 5592 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:42:13.315309 5603 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:13.439144 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:42:13.643099 1094 server/status/runtime.go:207 runtime stats: 359 MiB RSS, 106 goroutines, 17 MiB/12 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 36.60cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| W160906 11:42:13.822729 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:42:13.822953 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:42:13.869040 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:42:13.916220 5567 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:14.175806 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:14.243456 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:14.313513 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:14.315223 5621 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:14.358961 1223 server/status/runtime.go:207 runtime stats: 359 MiB RSS, 106 goroutines, 18 MiB/12 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 42.10cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:42:14.405560 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:14.433248 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:14.441053 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:14.441706 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:14.443913 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:14.713324 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:14.883771 5612 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:14.885576 5612 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 cluster-id system-db store:3 first-range] | |
| I160906 11:42:14.886301 5612 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:42:15.007440 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:15.495432 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:15.498504 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:15.499144 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:15.499361 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:15.809760 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:16.887756 5638 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:16.889815 5638 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:42:16.890552 5638 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 node:3 store:3 cluster-id] | |
| I160906 11:42:17.178277 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:17.313326 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:17.515010 5650 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:17.713290 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:18.007388 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:18.315022 5632 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:18.812291 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:18.889358 5658 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:18.890774 5658 gossip/client.go:189 [n3] node 3: sending [cluster-id node:1 node:2 node:3 store:3 first-range system-db] | |
| I160906 11:42:18.891362 5658 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 cluster-id system-db first-range] | |
| I160906 11:42:19.245238 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:19.407401 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:19.914835 1146 /go/src/google.golang.org/grpc/clientconn.go:667 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56887: getsockopt: connection refused"; Reconnecting to {"127.0.0.1:56887" <nil>} | |
| I160906 11:42:20.180648 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:20.313258 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:20.713237 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:20.890859 5672 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:20.893090 5672 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 cluster-id system-db first-range node:1 node:2] | |
| I160906 11:42:20.893797 5672 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:42:21.007340 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:21.067999 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:21.071317 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:21.071972 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:21.072236 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:21.114782 5693 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:21.814729 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:22.315403 5701 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:22.892448 5705 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:22.894594 5705 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:42:22.895310 5705 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:42:23.028944 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:23.037229 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:23.037918 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:23.039771 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:23.182983 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:23.313235 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:23.439119 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:42:23.643103 1094 server/status/runtime.go:207 runtime stats: 359 MiB RSS, 106 goroutines, 22 MiB/8.3 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.90cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:42:23.713228 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:23.715470 5730 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| W160906 11:42:23.822684 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:42:23.822899 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:42:23.868866 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.8 0.0 0.0 0.0 | |
| I160906 11:42:23.881705 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:23.884536 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:23.885185 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:23.885397 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:24.007343 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:24.246707 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:24.359230 1223 server/status/runtime.go:207 runtime stats: 360 MiB RSS, 106 goroutines, 23 MiB/8.0 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 30.60cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:42:24.408913 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:24.817165 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:24.893777 5735 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:24.895175 5735 gossip/client.go:189 [n3] node 3: sending [cluster-id node:1 node:2 node:3 store:3 first-range system-db] | |
| I160906 11:42:24.895772 5735 gossip/client.go:189 [n3] node 3: sending [store:3 cluster-id system-db first-range node:1 node:2 node:3] | |
| I160906 11:42:25.915245 5729 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:26.185350 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:26.313230 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:26.713241 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:26.895347 5763 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:26.896748 5763 gossip/client.go:189 [n3] node 3: sending [cluster-id node:1 node:2 node:3 store:3 first-range system-db] | |
| I160906 11:42:26.897403 5763 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:42:27.007550 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:27.115184 5774 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:27.819551 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:28.346150 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:28.349467 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:28.350120 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:28.350374 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:28.715213 5786 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:28.896395 5749 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:28.898150 5749 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:42:28.898722 5749 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range node:1] | |
| I160906 11:42:29.187937 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:29.248261 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:29.313327 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:29.410425 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:29.514893 5791 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:29.713495 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:30.007354 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:30.714856 5810 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:30.821789 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:30.898099 5815 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:30.900149 5815 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:42:30.900906 5815 gossip/client.go:189 [n3] node 3: sending [node:1 system-db cluster-id node:2 node:3 store:3 first-range] | |
| I160906 11:42:31.250548 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:31.259047 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:31.259675 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:31.261626 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:31.741003 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:31.744069 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:31.744754 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:31.744973 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:32.115153 5827 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:32.190213 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:32.313255 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:32.713254 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:32.899714 5805 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:32.901754 5805 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 cluster-id system-db first-range node:1 node:2] | |
| I160906 11:42:32.902461 5805 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:42:33.007354 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:33.439131 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:42:33.618049 1092 gossip/gossip.go:422 [n2] gossip status (stalled, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 42/37 sent/received, bytes 17181B/13485B sent/received) | |
| W160906 11:42:33.618507 1092 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| I160906 11:42:33.643008 1094 server/status/runtime.go:207 runtime stats: 362 MiB RSS, 106 goroutines, 27 MiB/4.7 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.90cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| W160906 11:42:33.823711 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:42:33.825064 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| W160906 11:42:33.825172 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:42:33.842782 1130 storage/store.go:1034 [n2,s2] error gossiping system config: node unavailable; try another peer | |
| I160906 11:42:33.845979 1129 storage/replica.go:2845 [n2,s2,r1{/Min-/Table/11}] gossiping cluster id {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} from store 2, range 1 | |
| W160906 11:42:33.846214 1129 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| W160906 11:42:33.846301 1129 storage/store.go:1014 [n2,s2] error gossiping first range data: node unavailable; try another peer | |
| I160906 11:42:33.869516 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:42:34.250053 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:34.358471 1221 gossip/gossip.go:422 [n3] gossip status (stalled, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 2654/32 sent/received, bytes 1333213B/12946B sent/received) | |
| I160906 11:42:34.360010 1223 server/status/runtime.go:207 runtime stats: 362 MiB RSS, 107 goroutines, 27 MiB/4.6 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 27.70cgo/sec, 1.01/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:42:34.412201 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:34.515208 5836 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:34.901249 5840 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:34.903321 5840 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 cluster-id system-db first-range] | |
| I160906 11:42:34.904078 5840 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:42:34.915046 5868 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:35.114892 5849 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:35.192564 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:35.313282 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:35.713454 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:35.937857 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:35.940727 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:35.941253 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:35.941446 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:36.007795 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:36.827371 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:36.902687 5876 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:36.904054 5876 gossip/client.go:189 [n3] node 3: sending [first-range system-db cluster-id node:1 node:2 node:3 store:3] | |
| I160906 11:42:36.904652 5876 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:42:37.915180 5889 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:38.194915 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:38.313277 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:38.713328 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:38.904413 5906 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:38.906261 5906 gossip/client.go:189 [n3] node 3: sending [first-range node:1 system-db cluster-id node:2 node:3 store:3] | |
| I160906 11:42:38.906927 5906 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 node:3 store:3 cluster-id] | |
| I160906 11:42:39.007419 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:39.251604 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:39.273033 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:39.276059 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:39.276722 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:39.276939 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:39.413717 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:39.715353 5894 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:39.829807 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:39.969772 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:39.976913 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:39.977462 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:39.979323 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:40.238662 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:42:40.906052 5900 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:40.907755 5900 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:42:40.908478 5900 gossip/client.go:189 [n3] node 3: sending [store:3 cluster-id system-db first-range node:1 node:2 node:3] | |
| I160906 11:42:41.197400 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:41.313539 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:41.316153 5945 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:41.713310 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:42.007464 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:42.832526 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:42.907801 5928 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:42.909292 5928 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:42:42.909883 5928 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:42:42.915432 5956 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:43.240935 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:43.243842 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:43.244433 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:43.244625 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:43.314456 5970 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:43.439174 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:42:43.643160 1094 server/status/runtime.go:207 runtime stats: 362 MiB RSS, 106 goroutines, 19 MiB/11 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 35.20cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| W160906 11:42:43.822728 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:42:43.822949 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:42:43.869076 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.8 0.0 0.0 0.0 | |
| I160906 11:42:44.037658 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:42:44.199983 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:44.253290 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:44.313351 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:44.359362 1223 server/status/runtime.go:207 runtime stats: 362 MiB RSS, 106 goroutines, 19 MiB/11 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 37.10cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:42:44.415214 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:44.713299 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:44.909780 5966 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:44.911459 5966 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 cluster-id system-db store:3 first-range node:1] | |
| I160906 11:42:44.912054 5966 gossip/client.go:189 [n3] node 3: sending [system-db node:3 store:3 first-range node:1 node:2 cluster-id] | |
| I160906 11:42:44.928774 5994 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:45.007357 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:45.514979 5998 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:45.835093 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:46.911447 6003 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:46.913263 6003 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:1 node:2 node:3 store:3 first-range] | |
| I160906 11:42:46.913870 6003 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:42:47.040661 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:47.043238 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:47.043780 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:47.043953 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:47.202785 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:47.313662 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:47.713279 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:47.791317 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:47.799282 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:47.799979 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:47.802212 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:48.007490 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:48.115369 5983 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:48.712237 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:42:48.716647 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:42:48.838366 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:48.913137 6019 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:48.914876 6019 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:42:48.916437 6019 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:42:48.922133 6030 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:49.255157 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:49.417078 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:49.714970 6040 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:50.205291 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:50.313323 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:50.713253 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:50.914949 6050 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:50.917195 6050 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:42:50.917914 6050 gossip/client.go:189 [n3] node 3: sending [store:3 cluster-id system-db first-range node:1 node:2 node:3] | |
| I160906 11:42:51.007450 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:51.719074 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:51.722645 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:51.723324 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:51.723542 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:51.840868 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:52.915440 6048 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:52.917980 6067 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:52.920294 6067 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:42:52.921040 6067 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:42:53.116816 6080 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:53.207172 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:53.313540 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:53.439150 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:42:53.643039 1094 server/status/runtime.go:207 runtime stats: 361 MiB RSS, 106 goroutines, 24 MiB/7.2 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.30cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:42:53.713519 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:42:53.822720 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:42:53.822941 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:42:53.869189 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.7 0.0 0.0 0.0 | |
| I160906 11:42:54.007423 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:54.256705 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:54.358913 1223 server/status/runtime.go:207 runtime stats: 361 MiB RSS, 106 goroutines, 24 MiB/7.0 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 36.50cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:42:54.418575 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:54.843962 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:54.921348 6087 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:54.922981 6087 gossip/client.go:189 [n3] node 3: sending [system-db store:3 first-range node:1 node:2 node:3 cluster-id] | |
| I160906 11:42:54.923564 6087 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:42:55.681440 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:55.684595 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:55.685260 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:55.685497 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:56.209677 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:56.315607 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:56.343238 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:56.350358 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:56.350945 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:56.353029 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:56.713301 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:56.922923 6103 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:56.924695 6103 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:42:56.925291 6103 gossip/client.go:189 [n3] node 3: sending [system-db store:3 first-range node:1 node:2 node:3 cluster-id] | |
| I160906 11:42:57.007429 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:57.315790 6120 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:57.846477 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:58.924452 6128 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:42:58.925965 6128 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:1 node:2 node:3 store:3 first-range] | |
| I160906 11:42:58.926576 6128 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:42:59.212226 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:42:59.258227 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:59.313303 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:42:59.420054 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:42:59.515493 6146 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:42:59.713312 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:42:59.838507 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:42:59.842027 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:42:59.842660 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:42:59.842901 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:00.007450 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:00.615445 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:00.618892 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:00.857959 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:00.925992 6145 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:00.927670 6145 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:43:00.928275 6145 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:43:01.315033 6173 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:02.214725 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:02.313557 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:02.315405 6158 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:02.713315 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:02.927616 6178 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:02.929438 6178 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:43:02.930183 6178 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 cluster-id system-db node:3] | |
| I160906 11:43:03.007439 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:03.439173 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:43:03.621874 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:03.625912 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:03.629366 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:03.629882 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:03.631692 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:03.633504 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:03.634025 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:03.634198 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:03.643174 1094 server/status/runtime.go:207 runtime stats: 362 MiB RSS, 106 goroutines, 29 MiB/3.8 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 32.80cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| W160906 11:43:03.822698 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:43:03.822898 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:43:03.860644 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:03.871303 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.5 0.0 0.0 0.0 | |
| I160906 11:43:04.260018 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:04.315697 6215 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:04.359215 1223 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 16 MiB/12 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.30cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:43:04.421343 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:04.437494 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:04.929612 6219 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:04.931059 6219 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:43:04.931650 6219 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 system-db cluster-id node:2 node:3] | |
| I160906 11:43:05.217171 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:05.313317 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:05.713525 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:05.715327 6231 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:06.007621 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:06.863055 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:06.931198 6202 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:06.932628 6202 gossip/client.go:189 [n3] node 3: sending [system-db store:3 first-range node:1 node:2 node:3 cluster-id] | |
| I160906 11:43:06.933216 6202 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 cluster-id system-db first-range] | |
| I160906 11:43:07.439938 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:07.443326 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:07.443873 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:07.444060 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:07.715361 6240 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:08.219534 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:08.313293 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:08.548034 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:08.713257 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:08.932766 6245 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:08.934510 6245 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 cluster-id system-db first-range] | |
| I160906 11:43:08.935111 6245 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:43:09.007379 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:09.261547 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:09.422804 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:09.715350 6257 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:09.865561 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:10.714920 6281 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:10.934329 6285 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:10.936050 6285 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:43:10.936790 6285 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:43:11.222052 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:11.313735 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:11.315779 6298 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:11.550689 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:11.553768 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:11.554415 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:11.554631 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:11.652839 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:11.661206 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:11.661869 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:11.663740 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:11.713250 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:11.982270 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:12.007441 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:12.868187 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:12.936029 6303 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:12.937884 6303 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:43:12.938608 6303 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:43:13.439164 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:43:13.643207 1094 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 20 MiB/10 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 37.50cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| W160906 11:43:13.822720 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:43:13.822938 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:43:13.869071 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.7 0.0 0.0 0.0 | |
| I160906 11:43:14.224668 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:14.263085 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:14.315425 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:14.359272 1223 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 20 MiB/10 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.90cgo/sec, 1.01/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:43:14.424297 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:14.713582 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:14.716312 6312 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:14.938029 6331 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:14.939656 6331 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:43:14.940410 6331 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:43:14.989036 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:14.992136 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:14.992688 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:14.992876 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:15.007374 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:15.514909 6317 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:15.870644 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:16.114765 6356 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:16.939596 6346 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:16.941011 6346 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:43:16.941597 6346 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:43:17.227160 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:17.313297 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:17.713297 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:18.007427 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:18.873175 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:18.941181 6362 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:18.942897 6362 gossip/client.go:189 [n3] node 3: sending [first-range system-db cluster-id node:1 node:2 node:3 store:3] | |
| I160906 11:43:18.943480 6362 gossip/client.go:189 [n3] node 3: sending [store:3 first-range system-db cluster-id node:1 node:2 node:3] | |
| I160906 11:43:19.264893 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:19.426045 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:19.715714 6375 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:19.763654 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:19.766559 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:19.767215 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:19.767433 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:20.229645 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:20.289928 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:20.297375 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:20.298063 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:20.300308 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:20.313423 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:20.515079 6385 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:20.713484 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:20.942772 6394 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:20.944848 6394 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:43:20.945567 6394 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 cluster-id system-db store:3 first-range] | |
| I160906 11:43:21.007352 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:21.875707 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:21.915368 6422 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:22.416443 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:22.419250 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:22.419799 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:22.419990 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:22.944429 6427 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:22.945804 6427 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:43:22.946379 6427 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:43:23.232179 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:23.313324 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:23.439158 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:43:23.515391 6440 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:23.643055 1094 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 24 MiB/6.8 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 31.50cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:43:23.713541 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:43:23.822734 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:43:23.822945 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:43:23.869196 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.8 0.0 0.0 0.0 | |
| I160906 11:43:24.012415 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:24.266452 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:24.358933 1223 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 25 MiB/6.7 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 32.80cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:43:24.427558 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:24.804244 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:24.878203 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:24.946426 6414 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:24.948159 6414 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range system-db cluster-id node:1 node:2] | |
| I160906 11:43:24.948927 6414 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 cluster-id system-db first-range node:1 node:2] | |
| I160906 11:43:25.515869 6445 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:26.234609 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:26.313318 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:26.713293 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:26.947960 6466 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:26.950044 6466 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range node:1] | |
| I160906 11:43:26.950640 6466 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:43:27.007425 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:27.715434 6478 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:27.806995 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:27.810450 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:27.811108 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:27.811328 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:27.869364 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:27.877190 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:27.877874 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:27.880107 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:27.882283 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:28.515083 6493 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:28.949570 6458 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:28.951279 6458 gossip/client.go:189 [n3] node 3: sending [cluster-id node:2 node:3 store:3 first-range node:1 system-db] | |
| I160906 11:43:28.951872 6458 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:43:29.237092 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:29.267529 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:29.313418 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:29.429135 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:29.713291 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:30.007619 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:30.171574 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:30.175165 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:30.175825 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:30.176042 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:30.884768 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:30.951227 6516 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:30.952684 6516 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 system-db cluster-id node:2] | |
| I160906 11:43:30.953283 6516 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:43:31.315012 6527 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:32.170741 1276 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:32.239534 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:32.313300 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:32.713279 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:32.952892 6535 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:32.954318 6535 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:43:32.954931 6535 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:43:33.007377 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:33.439161 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:43:33.514992 6508 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:33.618468 1092 gossip/gossip.go:422 [n2] gossip status (stalled, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 42/37 sent/received, bytes 17181B/13485B sent/received) | |
| W160906 11:43:33.618877 1092 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| I160906 11:43:33.643073 1094 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 29 MiB/3.2 MiB/42 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 32.00cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| W160906 11:43:33.822702 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:43:33.823128 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| W160906 11:43:33.842954 1130 storage/store.go:1034 [n2,s2] error gossiping system config: node unavailable; try another peer | |
| I160906 11:43:33.846050 1129 storage/replica.go:2845 [n2,s2,r1{/Min-/Table/11}] gossiping cluster id {"9ff0a99c-1fbf-4ac8-bea2-b09ffec2a124"} from store 2, range 1 | |
| W160906 11:43:33.846292 1129 gossip/infostore.go:295 [n2] node unavailable; try another peer | |
| W160906 11:43:33.846377 1129 storage/store.go:1014 [n2,s2] error gossiping first range data: node unavailable; try another peer | |
| I160906 11:43:33.883939 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.7 0.0 0.0 0.0 | |
| I160906 11:43:33.913922 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:34.117652 6547 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:34.268762 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:34.358944 1223 server/status/runtime.go:207 runtime stats: 365 MiB RSS, 106 goroutines, 16 MiB/14 MiB/43 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.80cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:43:34.359335 1221 gossip/gossip.go:422 [n3] gossip status (stalled, 3 nodes) | |
| gossip client (0/3 cur/max conns) | |
| gossip server (0/3 cur/max conns, infos 3074/32 sent/received, bytes 1565113B/12946B sent/received) | |
| I160906 11:43:34.430876 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:34.954323 6563 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:34.956080 6563 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:43:34.956856 6563 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:43:35.173680 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:35.176802 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:35.177487 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:35.177735 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:35.242069 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:35.313528 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:35.323356 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:35.330449 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:35.330996 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:35.332837 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:35.713274 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:36.007390 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:36.916425 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:36.955885 6586 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:36.957324 6586 gossip/client.go:189 [n3] node 3: sending [store:3 first-range system-db cluster-id node:1 node:2 node:3] | |
| I160906 11:43:36.957903 6586 gossip/client.go:189 [n3] node 3: sending [system-db first-range node:1 node:2 node:3 store:3 cluster-id] | |
| I160906 11:43:37.608693 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:37.612221 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:37.612900 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:37.613118 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:38.244514 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:38.313349 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:38.713291 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:38.916012 6557 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:38.957564 6610 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:38.958956 6610 gossip/client.go:189 [n3] node 3: sending [cluster-id node:1 node:2 node:3 store:3 first-range system-db] | |
| I160906 11:43:38.959537 6610 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:43:39.007625 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:39.114716 6598 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:39.270251 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:39.432419 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:39.918922 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:40.027544 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:40.959210 6603 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:40.960944 6603 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:43:40.961705 6603 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:43:41.246925 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:41.313322 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:41.713296 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:42.007480 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:42.715470 6633 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:42.921465 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:42.960855 6642 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:42.962504 6642 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:43:42.963184 6642 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 system-db cluster-id node:2] | |
| I160906 11:43:43.030585 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:43.039139 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:43.039767 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:43.041726 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:43.439143 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:43:43.514741 6637 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:43.643118 1094 server/status/runtime.go:207 runtime stats: 365 MiB RSS, 106 goroutines, 20 MiB/11 MiB/43 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.70cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| W160906 11:43:43.822722 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:43:43.822940 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:43:43.869059 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:43:43.889288 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:43.891694 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:43.892249 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:43.892443 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:44.249468 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:44.271779 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:44.313359 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:44.358891 1223 server/status/runtime.go:207 runtime stats: 365 MiB RSS, 106 goroutines, 20 MiB/11 MiB/43 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 30.50cgo/sec, 1.01/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:43:44.433978 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:44.515393 6680 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:44.713307 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:44.962447 6661 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:44.964519 6661 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 cluster-id system-db first-range] | |
| I160906 11:43:44.965241 6661 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:43:45.007446 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:45.924016 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:45.957724 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:45.961009 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:45.961688 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:45.961912 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:46.715449 6695 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:46.964579 6700 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:46.966322 6700 gossip/client.go:189 [n3] node 3: sending [store:3 first-range system-db cluster-id node:1 node:2 node:3] | |
| I160906 11:43:46.967086 6700 gossip/client.go:189 [n3] node 3: sending [system-db node:3 store:3 first-range node:1 node:2 cluster-id] | |
| I160906 11:43:47.252045 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:47.313296 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:47.584071 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:47.588266 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:47.713303 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:48.007394 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:48.315157 6714 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:48.926482 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:48.966135 6688 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:48.967801 6688 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 store:3 cluster-id system-db] | |
| I160906 11:43:48.968416 6688 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:43:49.273638 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:49.435798 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:50.254521 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:50.313328 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:50.591906 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:50.599119 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:50.599664 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:50.601576 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:50.713343 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:50.967873 6731 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:50.969372 6731 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:43:50.970004 6731 gossip/client.go:189 [n3] node 3: sending [store:3 first-range node:1 node:2 node:3 cluster-id system-db] | |
| I160906 11:43:51.007447 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:51.317722 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:51.715389 6741 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:51.928945 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:52.005985 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:52.009851 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:52.010393 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:52.010563 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:52.515339 6767 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:52.969451 6745 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:52.970868 6745 gossip/client.go:189 [n3] node 3: sending [cluster-id node:2 node:3 store:3 first-range node:1 system-db] | |
| I160906 11:43:52.971491 6745 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:43:53.256964 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:53.313341 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:53.439173 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:43:53.643047 1094 server/status/runtime.go:207 runtime stats: 365 MiB RSS, 106 goroutines, 25 MiB/7.7 MiB/43 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 35.60cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:43:53.713236 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| W160906 11:43:53.822719 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:43:53.822922 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:43:53.869126 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.6 0.0 0.0 0.0 | |
| I160906 11:43:54.008926 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:54.275144 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:54.320195 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:54.323678 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:54.324364 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:54.324617 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:54.358999 1223 server/status/runtime.go:207 runtime stats: 365 MiB RSS, 106 goroutines, 25 MiB/7.5 MiB/43 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 35.60cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:43:54.437330 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:54.931518 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:54.971485 6777 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:54.973472 6777 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 node:3 cluster-id system-db store:3] | |
| I160906 11:43:54.974193 6777 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range node:1] | |
| I160906 11:43:55.515366 6805 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:55.749793 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:56.259447 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:56.313303 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:56.713327 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:56.716326 6812 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:56.973086 6791 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:56.974560 6791 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 cluster-id system-db store:3 first-range] | |
| I160906 11:43:56.975294 6791 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range node:1] | |
| I160906 11:43:57.007413 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:57.315018 6817 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:43:57.934078 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:58.754011 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:43:58.762605 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:43:58.763261 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:58.765182 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:43:58.974730 6840 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:43:58.976438 6840 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range system-db cluster-id node:1 node:2] | |
| I160906 11:43:58.977155 6840 gossip/client.go:189 [n3] node 3: sending [first-range node:1 system-db cluster-id node:2 node:3 store:3] | |
| I160906 11:43:59.261916 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:43:59.276682 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:59.313316 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:43:59.438813 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:43:59.456458 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:43:59.713325 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:43:59.716328 6854 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:00.007477 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:00.440161 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:44:00.444075 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:44:00.444734 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:00.444949 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:00.915398 6823 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:00.936435 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:00.976258 6828 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:00.977635 6828 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:44:00.978233 6828 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 node:2 cluster-id system-db] | |
| I160906 11:44:01.714990 6872 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:02.264357 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:02.313345 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:02.460350 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:44:02.480213 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:44:02.481033 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:02.481341 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:02.713579 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:44:02.715668 6880 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:02.977968 6900 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:02.979679 6900 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 cluster-id system-db store:3 first-range] | |
| I160906 11:44:02.980406 6900 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db first-range node:1 node:2 node:3 store:3] | |
| I160906 11:44:03.007407 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:03.439149 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:44:03.643052 1094 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 16 MiB/12 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 34.20cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| W160906 11:44:03.822701 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:44:03.822931 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:44:03.872409 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.7 0.0 0.0 0.0 | |
| I160906 11:44:03.938964 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:04.176297 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:44:04.278456 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:44:04.359300 1223 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 16 MiB/11 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 36.60cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (1x) | |
| I160906 11:44:04.440679 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:44:04.715388 6883 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:04.979560 6888 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:04.981601 6888 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 cluster-id system-db first-range node:1] | |
| I160906 11:44:04.982328 6888 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 first-range system-db cluster-id] | |
| I160906 11:44:05.266880 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:05.313547 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:05.315648 6932 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:05.713303 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:44:06.007404 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:06.315349 6937 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:06.941330 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:06.981110 6917 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:06.982473 6917 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 system-db cluster-id node:2] | |
| I160906 11:44:06.983057 6917 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 cluster-id system-db first-range] | |
| I160906 11:44:07.179276 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:44:07.187507 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:44:07.188185 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:07.190292 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:07.197042 1225 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:44:08.269318 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:08.313323 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:08.315159 6963 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:08.588927 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:44:08.591491 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:44:08.592035 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:08.592249 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:08.713320 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:44:08.982669 6971 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:08.984418 6971 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:2 node:3 store:3 first-range node:1] | |
| I160906 11:44:08.985172 6971 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db store:3 first-range node:1 node:2 node:3] | |
| I160906 11:44:09.007404 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:09.280014 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:44:09.442292 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:44:09.915372 6949 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:09.943703 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:10.200174 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:44:10.203400 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:44:10.204036 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:10.204262 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:10.514729 6989 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:10.984311 6993 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:10.986035 6993 gossip/client.go:189 [n3] node 3: sending [node:1 system-db cluster-id node:2 node:3 store:3 first-range] | |
| I160906 11:44:10.986743 6993 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:44:11.271688 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:11.313298 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:11.713327 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:44:12.007460 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:12.830035 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:44:12.915580 6956 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:12.946202 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:12.986507 6961 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:12.988115 6961 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range node:1 system-db cluster-id node:2] | |
| I160906 11:44:12.988701 6961 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:1 node:2 node:3 store:3 first-range] | |
| I160906 11:44:13.114824 7020 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:13.439170 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:44:13.643044 1094 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 22 MiB/7.5 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 43.50cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| W160906 11:44:13.822721 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:44:13.822945 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:44:13.869053 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.8 0.0 0.0 0.0 | |
| I160906 11:44:14.274342 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:14.281578 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:44:14.313289 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:14.358888 1223 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 22 MiB/7.4 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 41.10cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:44:14.444449 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:44:14.713301 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:44:14.991880 7007 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:14.995245 7007 gossip/client.go:189 [n3] node 3: sending [first-range node:1 node:2 cluster-id system-db node:3 store:3] | |
| I160906 11:44:14.995986 7007 gossip/client.go:189 [n3] node 3: sending [system-db cluster-id node:1 node:2 node:3 store:3 first-range] | |
| I160906 11:44:15.007384 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:15.833490 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:44:15.842022 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:44:15.842656 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:15.844533 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:15.948811 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:16.025859 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:44:16.028930 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:44:16.029586 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:16.029805 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:16.915383 7053 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:16.993474 7057 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:16.995140 7057 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:44:16.995715 7057 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 cluster-id system-db node:3 store:3 first-range] | |
| I160906 11:44:17.276868 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:17.313297 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:17.713260 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:44:17.716274 7071 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:18.007372 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:18.238734 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:44:18.241754 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:44:18.242288 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:18.242487 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:18.951564 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:18.994916 7081 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:18.996332 7081 gossip/client.go:189 [n3] node 3: sending [node:3 store:3 first-range system-db cluster-id node:1 node:2] | |
| I160906 11:44:18.996904 7081 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:44:19.283390 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:44:19.446325 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:44:20.279453 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:20.313283 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:20.555315 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:44:20.558892 1224 kv/dist_sender.go:1036 [n3] application error: node unavailable; try another peer | |
| I160906 11:44:20.713529 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:44:20.715586 7096 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:20.996624 7039 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:20.998791 7039 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:44:20.999528 7039 gossip/client.go:189 [n3] node 3: sending [node:2 cluster-id system-db node:3 store:3 first-range node:1] | |
| I160906 11:44:21.007393 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:21.954085 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:22.514855 7103 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:22.998243 7114 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:22.999949 7114 gossip/client.go:189 [n3] node 3: sending [node:2 node:3 store:3 first-range node:1 system-db cluster-id] | |
| I160906 11:44:23.000686 7114 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 first-range system-db cluster-id] | |
| I160906 11:44:23.281924 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:23.313306 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:23.439159 1037 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 2 0 0.0 0.0 0.0 0.0 | |
| 3: 0 1 0 0.0 0.0 0.0 0.0 | |
| I160906 11:44:23.562306 1224 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:44:23.570441 1224 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:44:23.571132 1224 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:23.573395 1224 kv/dist_sender.go:771 reply error Merge [/System/tsd/cr.node.gossip.connections.incoming/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.bytes.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.received-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-10m/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.infos.sent-1h/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.gossip.connections.outgoing/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-count/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.node.txn.aborts-1m/3/10s/2016-09-06T11:00:00Z,/Min), ... 190 skipped ..., Merge [/System/tsd/cr.store.raft.ticks/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.enqueued.pending/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.raft.leaders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders/3/10s/2016-09-06T11:00:00Z,/Min), Merge [/System/tsd/cr.store.range.leaseholders.without.leadership/3/10s/2016-09-06T11:00:00Z,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:23.643073 1094 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 26 MiB/3.9 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 30.50cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:44:23.713311 1242 storage/replica.go:1950 [n3,s3,r1{/Min-/Table/11}] pending commands: refurbished 1, reproposing 0 (at 88.72); reasonTicks | |
| I160906 11:44:23.790361 1276 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:44:23.792945 1276 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:44:23.793478 1276 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:23.793656 1276 kv/dist_sender.go:771 reply error RangeLookup [/Meta2/Table/13,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| W160906 11:44:23.822714 1095 ts/db.go:104 node unavailable; try another peer | |
| W160906 11:44:23.822937 1096 server/node.go:670 [n2] error recording status summaries: node unavailable; try another peer | |
| I160906 11:44:23.868882 1115 storage/raft_transport.go:217 stats: | |
| qlen qmax qdropped client-sent client-recv server-sent server-recv | |
| 1: 0 4 0 0.0 0.0 0.0 0.0 | |
| 2: 0 2 0 0.4 0.0 0.0 0.0 | |
| I160906 11:44:24.007388 1122 storage/replica.go:1950 [n2,s2,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| I160906 11:44:24.284889 1036 kv/txn_coord_sender.go:251 [n2] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:44:24.358949 1223 server/status/runtime.go:207 runtime stats: 363 MiB RSS, 106 goroutines, 27 MiB/3.8 MiB/40 MiB GO alloc/idle/total, 45 MiB/59 MiB CGO alloc/total, 30.50cgo/sec, 1.02/0.00 %(u/s)time, 0.00 %gc (0x) | |
| I160906 11:44:24.447875 1114 kv/txn_coord_sender.go:251 [n3] txn coordinator: 0.00 txn/sec, 100.00/0.00/0.00/0.00 %cmmt/cmmt1pc/abrt/abnd, 0s/0s/0s avg/σ/max duration, 0.0/0.0/0 avg/σ/max restarts (0 samples) | |
| I160906 11:44:24.514944 7131 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:24.956560 1438 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:24.999832 7136 gossip/client.go:125 [n3] node 3: started gossip client to 127.0.0.1:41417 | |
| I160906 11:44:25.001605 7136 gossip/client.go:189 [n3] node 3: sending [node:1 node:2 node:3 store:3 cluster-id system-db first-range] | |
| I160906 11:44:25.002183 7136 gossip/client.go:189 [n3] node 3: sending [cluster-id system-db node:3 store:3 first-range node:1 node:2] | |
| I160906 11:44:25.606370 1225 kv/dist_sender.go:1036 application error: range 1: replica {3 3 3} not lease holder; <nil> is | |
| I160906 11:44:25.610403 1225 kv/dist_sender.go:1036 application error: node unavailable; try another peer | |
| W160906 11:44:25.611116 1225 kv/dist_sender.go:1052 RPC error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:25.611336 1225 kv/dist_sender.go:771 reply error Put [/System/StatusNode/3,/Min): failed to send RPC: sending to all 3 replicas failed; last error: rpc error: code = 14 desc = grpc: the connection is unavailable | |
| I160906 11:44:25.921239 7150 storage/raft_transport.go:405 raft transport stream to node 2 established | |
| I160906 11:44:26.284502 1233 rpc/clock_offset.go:147 2 of 2 nodes are within the maximum offset of 50ms | |
| I160906 11:44:26.313306 1242 storage/replica.go:1950 [n3,s3,r4{/Table/13-/Table/14}] pending commands: refurbished 1, reproposing 0 (at 38.25); reasonTicks | |
| panic: test timed out after 8m0s | |
| goroutine 7163 [running]: | |
| panic(0x1954ba0, 0xc421975ae0) | |
| /usr/local/go/src/runtime/panic.go:500 +0x1ae | |
| testing.startAlarm.func1() | |
| /usr/local/go/src/testing/testing.go:918 +0x14e | |
| created by time.goFunc | |
| /usr/local/go/src/time/sleep.go:154 +0x78 | |
| goroutine 1 [chan receive, 7 minutes]: | |
| testing.(*T).Run(0xc420249500, 0x1af8af5, 0x1a, 0x1d79658, 0x70a301) | |
| /usr/local/go/src/testing/testing.go:647 +0x56e | |
| testing.RunTests.func1(0xc420249500) | |
| /usr/local/go/src/testing/testing.go:793 +0xba | |
| testing.tRunner(0xc420249500, 0xc4206c3d78) | |
| /usr/local/go/src/testing/testing.go:610 +0xca | |
| testing.RunTests(0x1d7a728, 0x25fc7a0, 0x4, 0x4, 0x0) | |
| /usr/local/go/src/testing/testing.go:799 +0x4bb | |
| testing.(*M).Run(0xc4206c3ef0, 0x331ff00) | |
| /usr/local/go/src/testing/testing.go:743 +0x130 | |
| github.com/cockroachdb/cockroach/testutils/testcluster.TestMain(0xc4206c3ef0) | |
| /go/src/github.com/cockroachdb/cockroach/testutils/testcluster/main_test.go:34 +0xd7 | |
| main.main() | |
| github.com/cockroachdb/cockroach/testutils/testcluster/_test/_testmain.go:58 +0x1b6 | |
| goroutine 17 [syscall, 7 minutes, locked to thread]: | |
| runtime.goexit() | |
| /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1 | |
| goroutine 6 [chan receive]: | |
| github.com/cockroachdb/cockroach/util/log.(*loggingT).flushDaemon(0x29640e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/log/clog.go:996 +0x85 | |
| created by github.com/cockroachdb/cockroach/util/log.init.1 | |
| /go/src/github.com/cockroachdb/cockroach/util/log/clog.go:577 +0xc3 | |
| goroutine 1225 [select]: | |
| github.com/cockroachdb/cockroach/util/retry.(*Retry).Next(0xc4221a27c0, 0x7fc78fa10408) | |
| /go/src/github.com/cockroachdb/cockroach/util/retry/retry.go:128 +0x307 | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk(0xc420a0fdd0, 0x7fc78fa10408, 0xc4215fbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:683 +0x348 | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).Send(0xc420a0fdd0, 0x7fc78fa10408, 0xc4215fbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:606 +0x1a1 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc420a09c30, 0x7fc78fa10408, 0xc4215fbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:391 +0x68a | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).send(0xc42101c780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:495 +0x294 | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).(github.com/cockroachdb/cockroach/internal/client.send)-fm(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:440 +0x7d | |
| github.com/cockroachdb/cockroach/internal/client.sendAndFill(0xc4221a39c0, 0xc420ecec00, 0x0, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:422 +0x1ac | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).Run(0xc42101c780, 0xc420ecec00, 0xc421e61b10, 0x76fc38) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:440 +0xd5 | |
| github.com/cockroachdb/cockroach/internal/client.runOneResult(0x260a6c0, 0xc42101c780, 0xc420ecec00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:522 +0x98 | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).PutInline(0xc42101c780, 0x1a3eda0, 0xc421a82a80, 0x1a4fb60, 0xc421445ad0, 0xc4207dc6f0, 0xc421e61c70) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:250 +0xf7 | |
| github.com/cockroachdb/cockroach/server.(*Node).writeSummaries.func1() | |
| /go/src/github.com/cockroachdb/cockroach/server/node.go:694 +0x182 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc4207dc6c0, 0xc4221a3db8, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:218 +0x10e | |
| github.com/cockroachdb/cockroach/server.(*Node).writeSummaries(0xc4212086c0, 0xc4206e3b60, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/server/node.go:705 +0xb7 | |
| github.com/cockroachdb/cockroach/server.(*Node).startWriteSummaries.func1() | |
| /go/src/github.com/cockroachdb/cockroach/server/node.go:669 +0x1b2 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc421551740) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 927 [semacquire, 7 minutes]: | |
| sync.runtime_notifyListWait(0xc420ec5610, 0xc400000002) | |
| /usr/local/go/src/runtime/sema.go:267 +0x12f | |
| sync.(*Cond).Wait(0xc420ec5600) | |
| /usr/local/go/src/sync/cond.go:57 +0x8e | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).Quiesce(0xc420c4d830) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:416 +0x2a4 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).Stop(0xc420c4d830) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:359 +0x355 | |
| github.com/cockroachdb/cockroach/testutils/testcluster.(*TestCluster).stopServers(0xc42144c4e0) | |
| /go/src/github.com/cockroachdb/cockroach/testutils/testcluster/testcluster.go:84 +0x188 | |
| github.com/cockroachdb/cockroach/testutils/testcluster.(*TestCluster).(github.com/cockroachdb/cockroach/testutils/testcluster.stopServers)-fm() | |
| /go/src/github.com/cockroachdb/cockroach/testutils/testcluster/testcluster.go:159 +0x42 | |
| github.com/cockroachdb/cockroach/util/stop.CloserFn.Close(0xc4212f68b0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:86 +0x35 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).Stop(0xc4214801b0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:365 +0x435 | |
| github.com/cockroachdb/cockroach/testutils/testcluster.TestWaitForFullReplication(0xc42146c180) | |
| /go/src/github.com/cockroachdb/cockroach/testutils/testcluster/testcluster_test.go:180 +0x132 | |
| testing.tRunner(0xc42146c180, 0x1d79658) | |
| /usr/local/go/src/testing/testing.go:610 +0xca | |
| created by testing.(*T).Run | |
| /usr/local/go/src/testing/testing.go:646 +0x530 | |
| goroutine 1220 [select]: | |
| github.com/cockroachdb/cockroach/server.(*Node).startComputePeriodicMetrics.func1() | |
| /go/src/github.com/cockroachdb/cockroach/server/node.go:631 +0x2f3 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc421551660) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1142 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc421047700, 0xecf60a141, 0xc432774c9b, 0x2962d60, 0xc420b51e80, 0xc420c4d830, 0x0, 0xc4214492c0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x299 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x540 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420da0b80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1092 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/server.(*Node).startGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/server/node.go:591 +0x545 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc4208908a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1459 [IO wait]: | |
| net.runtime_pollWait(0x7fc78fa3d6d0, 0x72, 0x5f17b3) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x5e | |
| net.(*pollDesc).wait(0xc420f80ca0, 0x72, 0x5ee068, 0x19c4020) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b | |
| net.(*pollDesc).waitRead(0xc420f80ca0, 0x26118c0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x42 | |
| net.(*netFD).Read(0xc420f80c40, 0xc421333000, 0x400, 0x400, 0x0, 0x26118c0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1d9 | |
| net.(*conn).Read(0xc4200261b0, 0xc421333000, 0x400, 0x400, 0x102, 0xc42133300d, 0xc421333005) | |
| /usr/local/go/src/net/net.go:173 +0x97 | |
| net.(*TCPConn).Read(0xc4200261b0, 0xc421333000, 0x400, 0x400, 0xc421bfb540, 0xba84a9, 0xc421bfb540) | |
| <autogenerated>:69 +0x73 | |
| crypto/tls.(*block).readFromUntil(0xc421aff740, 0x7fc78fa16c70, 0xc4200261b0, 0x5, 0xc4200261b0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:472 +0x10d | |
| crypto/tls.(*Conn).readRecord(0xc420450000, 0x1d7b117, 0xc420450100, 0xc421bfb9a8) | |
| /usr/local/go/src/crypto/tls/conn.go:574 +0x13d | |
| crypto/tls.(*Conn).Read(0xc420450000, 0xc421b36000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1109 +0x186 | |
| bufio.(*Reader).fill(0xc421439b60) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x197 | |
| bufio.(*Reader).Read(0xc421439b60, 0xc4217a3178, 0x9, 0x9, 0x0, 0x0, 0xc421bfbb48) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x6ac | |
| io.ReadAtLeast(0x2609f40, 0xc421439b60, 0xc4217a3178, 0x9, 0x9, 0x9, 0xc421186218, 0xc4200641a0, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xb2 | |
| io.ReadFull(0x2609f40, 0xc421439b60, 0xc4217a3178, 0x9, 0x9, 0x7fc78fb164b0, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x73 | |
| golang.org/x/net/http2.readFrameHeader(0xc4217a3178, 0x9, 0x9, 0x2609f40, 0xc421439b60, 0x0, 0x0, 0xc420ee5590, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0xa3 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc4217a3140, 0x2, 0xc421d45770, 0x22, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0x105 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc42141cc00, 0xc4211869c0, 0xc4211869c0, 0xc421bfbec0, 0x1) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53 | |
| google.golang.org/grpc/transport.(*http2Client).reader(0xc420f83000) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:921 +0x154 | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:203 +0xc1a | |
| goroutine 1121 [IO wait, 7 minutes]: | |
| net.runtime_pollWait(0x7fc78fa3d790, 0x72, 0x5f17b3) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x5e | |
| net.(*pollDesc).wait(0xc421150300, 0x72, 0x5ee068, 0x19c4020) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b | |
| net.(*pollDesc).waitRead(0xc421150300, 0x26118c0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x42 | |
| net.(*netFD).accept(0xc4211502a0, 0x0, 0x260ec80, 0xc4210c9d40) | |
| /usr/local/go/src/net/fd_unix.go:419 +0x2b8 | |
| net.(*TCPListener).accept(0xc420026428, 0xc420aefed8, 0x60c373, 0xc420aefea8) | |
| /usr/local/go/src/net/tcpsock_posix.go:132 +0x51 | |
| net.(*TCPListener).Accept(0xc420026428, 0x1d77fc0, 0xc420c01e00, 0xc421402f20, 0x643f72) | |
| /usr/local/go/src/net/tcpsock.go:222 +0x50 | |
| github.com/cockroachdb/cmux.(*cMux).Serve(0xc420c01e00, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cmux/cmux.go:124 +0xdd | |
| github.com/cockroachdb/cockroach/server.(*Server).Start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/server/server.go:392 +0x47 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c98e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1116 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/sql.(*LeaseManager).RefreshLeases.func1() | |
| /go/src/github.com/cockroachdb/cockroach/sql/lease.go:1144 +0x1204 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc42119c360) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1246 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421040900) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x58 | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xd8 | |
| goroutine 1139 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420da0aa0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1115 [select]: | |
| github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0x12f6 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc421431dc0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1141 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420da0b60) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1242 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2778 +0x184a | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc421303020) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1159 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cockroach/gossip.(*server).start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/server.go:370 +0x94 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c9a00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1094 [select]: | |
| github.com/cockroachdb/cockroach/server.(*Server).startSampleEnvironment.func1() | |
| /go/src/github.com/cockroachdb/cockroach/server/server.go:626 +0x1b4 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc4201ba000) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1157 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4210c97e0, 0x1d7a498, 0xc420ddf360, 0x2624180, 0xc4211fd290) | |
| <autogenerated>:9 +0xa4 | |
| google.golang.org/grpc.(*Server).Serve(0xc420ddf360, 0x261c440, 0xc4210c97e0, 0x0, 0x0) | |
| /go/src/google.golang.org/grpc/server.go:348 +0x1b8 | |
| github.com/cockroachdb/cockroach/server.(*Server).Start.func7() | |
| /go/src/github.com/cockroachdb/cockroach/server/server.go:414 +0x8d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c9980) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1156 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cockroach/server.(*Server).Start.func6() | |
| /go/src/github.com/cockroachdb/cockroach/server/server.go:407 +0xa6 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c9960) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1158 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4210c97c0, 0x10, 0xc421402f30, 0x7fc78fa39170, 0x0) | |
| <autogenerated>:9 +0xa4 | |
| github.com/cockroachdb/cockroach/util/netutil.(*Server).ServeWith(0xc420026408, 0xc4207dc6c0, 0x261c440, 0xc4210c97c0, 0xc421402f30, 0xa20a0d, 0xc420aaef88) | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:117 +0x71 | |
| github.com/cockroachdb/cockroach/server.(*Server).Start.func8() | |
| /go/src/github.com/cockroachdb/cockroach/server/server.go:422 +0x112 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc42119d020) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1073 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*bookie).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x4c8 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc421352080) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1113 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:340 +0x519 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c9280) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1244 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421040900) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x58 | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xd8 | |
| goroutine 1277 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc420d60de0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1438 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc420a14580, 0xc42000b100, 0xc4211e4190, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x813 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0xb3 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc42149a2a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1126 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42138d6e0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x58 | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xd8 | |
| goroutine 1038 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/sql.(*LeaseManager).RefreshLeases.func1() | |
| /go/src/github.com/cockroachdb/cockroach/sql/lease.go:1144 +0x1204 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420b63980) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1036 [select]: | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc420a30dc0) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xde7 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm() | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x42 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420374900) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1252 [chan receive, 7 minutes]: | |
| database/sql.(*DB).connectionOpener(0xc420d32580) | |
| /usr/local/go/src/database/sql/sql.go:730 +0x6e | |
| created by database/sql.Open | |
| /usr/local/go/src/database/sql/sql.go:493 +0x2af | |
| goroutine 1118 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cockroach/util/netutil.MakeServer.func2() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:100 +0x8e | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c9680) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 5150 [select, 3 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*Replica).addWriteCmd(0xc4201ae900, 0x7fc78fa10408, 0xc420eaa780, 0x1471b91ebd234285, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:1457 +0x4d5 | |
| github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc4201ae900, 0x7fc78fa10408, 0xc420eaa780, 0x1471b91ebd234285, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:1001 +0x2a5 | |
| github.com/cockroachdb/cockroach/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica_range_lease.go:125 +0x22d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask.func1(0xc4207dc6c0, 0x230c2b4, 0x1e, 0x8f, 0xc420eaa690) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:234 +0xa3 | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:235 +0xca | |
| goroutine 1161 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x416 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc421402870) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1224 [select]: | |
| github.com/cockroachdb/cockroach/util/retry.(*Retry).Next(0xc4210b8808, 0x7fc78fa10408) | |
| /go/src/github.com/cockroachdb/cockroach/util/retry/retry.go:128 +0x307 | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk(0xc420a0fdd0, 0x7fc78fa10408, 0xc4218aa6f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:683 +0x348 | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).Send(0xc420a0fdd0, 0x7fc78fa10408, 0xc4218aa6f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:606 +0x1a1 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc420a09c30, 0x7fc78fa10408, 0xc4218aa6f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:391 +0x68a | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).send(0xc42101c780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:495 +0x294 | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).(github.com/cockroachdb/cockroach/internal/client.send)-fm(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:440 +0x7d | |
| github.com/cockroachdb/cockroach/internal/client.sendAndFill(0xc4210b9a08, 0xc420c9e000, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:422 +0x1ac | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).Run(0xc42101c780, 0xc420c9e000, 0x1, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:440 +0xd5 | |
| github.com/cockroachdb/cockroach/ts.(*DB).StoreData(0xc420026400, 0x1, 0xc420237000, 0xd7, 0xd7, 0xa1f120, 0xc4207dc6f0) | |
| /go/src/github.com/cockroachdb/cockroach/ts/db.go:143 +0x8fa | |
| github.com/cockroachdb/cockroach/ts.(*poller).poll.func1() | |
| /go/src/github.com/cockroachdb/cockroach/ts/db.go:100 +0x11b | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc4207dc6c0, 0xc4210b9e48, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:218 +0x10e | |
| github.com/cockroachdb/cockroach/ts.(*poller).poll(0xc42141ce70) | |
| /go/src/github.com/cockroachdb/cockroach/ts/db.go:103 +0x9e | |
| github.com/cockroachdb/cockroach/ts.(*poller).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/ts/db.go:83 +0x16f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4211c33a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1451 [IO wait]: | |
| net.runtime_pollWait(0x7fc78fa10b50, 0x72, 0x5f17b3) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x5e | |
| net.(*pollDesc).wait(0xc420e744c0, 0x72, 0x5ee068, 0x19c4020) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b | |
| net.(*pollDesc).waitRead(0xc420e744c0, 0x26118c0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x42 | |
| net.(*netFD).Read(0xc420e74460, 0xc4218b2000, 0x8000, 0x8000, 0x0, 0x26118c0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1d9 | |
| net.(*conn).Read(0xc420026140, 0xc4218b2000, 0x8000, 0x8000, 0x88, 0x0, 0x1d7aee8) | |
| /usr/local/go/src/net/net.go:173 +0x97 | |
| net.(*TCPConn).Read(0xc420026140, 0xc4218b2000, 0x8000, 0x8000, 0x1, 0x260a4c0, 0xc42007c020) | |
| <autogenerated>:69 +0x73 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42037cfd0, 0xc4218b2000, 0x8000, 0x8000, 0xc4204e9328, 0xba84a9, 0xc4204e9328) | |
| /go/src/github.com/cockroachdb/cmux/cmux.go:218 +0x183 | |
| crypto/tls.(*block).readFromUntil(0xc42121bd40, 0x7fc78f9c9648, 0xc42037cfd0, 0x5, 0xc42037cfd0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:472 +0x10d | |
| crypto/tls.(*Conn).readRecord(0xc420450380, 0x1d7b117, 0xc420450480, 0xc4218b200d) | |
| /usr/local/go/src/crypto/tls/conn.go:574 +0x13d | |
| crypto/tls.(*Conn).Read(0xc420450380, 0xc421942000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1109 +0x186 | |
| bufio.(*Reader).fill(0xc421415200) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x197 | |
| bufio.(*Reader).Read(0xc421415200, 0xc420f51538, 0x9, 0x9, 0xc42193f170, 0x5ef0c7, 0xc42112bdf8) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x6ac | |
| io.ReadAtLeast(0x2609f40, 0xc421415200, 0xc420f51538, 0x9, 0x9, 0x9, 0x61c2ba, 0xc4202e7040, 0xc4211866c8) | |
| /usr/local/go/src/io/io.go:307 +0xb2 | |
| io.ReadFull(0x2609f40, 0xc421415200, 0xc420f51538, 0x9, 0x9, 0x0, 0xc4213d59f0, 0xc4204e9a10) | |
| /usr/local/go/src/io/io.go:325 +0x73 | |
| golang.org/x/net/http2.readFrameHeader(0xc420f51538, 0x9, 0x9, 0x2609f40, 0xc421415200, 0x0, 0xc400000000, 0xc4211866c8, 0x260dec0) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0xa3 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc420f51500, 0x26137c0, 0xc4211868a0, 0x0, 0xc421388dc0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0x105 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc42191e450, 0xc4211868a0, 0xc4211868a0, 0xc4204e9cd0, 0x1) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53 | |
| google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc421480090, 0xc42191e4e0) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x30a | |
| google.golang.org/grpc.(*Server).serveStreams(0xc4206f9d60, 0x2623e80, 0xc421480090) | |
| /go/src/google.golang.org/grpc/server.go:424 +0x1d6 | |
| google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc4206f9d60, 0x2624120, 0xc420450380, 0x2610b00, 0xc42145c420) | |
| /go/src/google.golang.org/grpc/server.go:411 +0x4f1 | |
| google.golang.org/grpc.(*Server).handleRawConn(0xc4206f9d60, 0x2624180, 0xc42037cfd0) | |
| /go/src/google.golang.org/grpc/server.go:388 +0x5c3 | |
| created by google.golang.org/grpc.(*Server).Serve | |
| /go/src/google.golang.org/grpc/server.go:357 +0x20a | |
| goroutine 1112 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cockroach/rpc.NewContext.func1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:126 +0x95 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c8960) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1711 [runnable]: | |
| encoding/gob.encStructTerminator(0xc4206fd9e0, 0xc421a95080, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/encoding/gob/encode.go:268 | |
| encoding/gob.(*Encoder).encodeStruct(0xc4213c6640, 0xc4213c6678, 0xc4200ce980, 0x1a428e0, 0xc421aa8960, 0x199) | |
| /usr/local/go/src/encoding/gob/encode.go:323 +0x55c | |
| encoding/gob.encOpFor.func4(0xc4203803c0, 0xc421a95000, 0x1a428e0, 0xc421aa8960, 0x199) | |
| /usr/local/go/src/encoding/gob/encode.go:587 +0x117 | |
| encoding/gob.(*Encoder).encodeStruct(0xc4213c6640, 0xc4213c6678, 0xc4203fe540, 0x1a42520, 0xc421aa8960, 0x199) | |
| /usr/local/go/src/encoding/gob/encode.go:334 +0x438 | |
| encoding/gob.encOpFor.func4(0xc42037e360, 0xc421a94f80, 0x1a42520, 0xc421aa8960, 0x199) | |
| /usr/local/go/src/encoding/gob/encode.go:587 +0x117 | |
| encoding/gob.(*Encoder).encodeStruct(0xc4213c6640, 0xc4213c6678, 0xc4200ce9e0, 0x1ad61e0, 0xc4220367d0, 0x199) | |
| /usr/local/go/src/encoding/gob/encode.go:334 +0x438 | |
| encoding/gob.encOpFor.func4(0x0, 0xc421a94f40, 0x1ad61e0, 0xc4220367d0, 0x199) | |
| /usr/local/go/src/encoding/gob/encode.go:587 +0x117 | |
| encoding/gob.(*Encoder).encodeArray(0xc4213c6640, 0xc4213c6678, 0x193e200, 0xc42087e528, 0x197, 0xc4203740c0, 0x0, 0xd7, 0x0) | |
| /usr/local/go/src/encoding/gob/encode.go:357 +0x2ca | |
| encoding/gob.encOpFor.func1(0xc420380cf0, 0xc421a94f00, 0x193e200, 0xc42087e528, 0x197) | |
| /usr/local/go/src/encoding/gob/encode.go:557 +0x171 | |
| encoding/gob.(*Encoder).encodeStruct(0xc4213c6640, 0xc4213c6678, 0xc4200ce8a0, 0x1a41f20, 0xc42087e4d0, 0x199) | |
| /usr/local/go/src/encoding/gob/encode.go:334 +0x438 | |
| encoding/gob.(*Encoder).encode(0xc4213c6640, 0xc4213c6678, 0x1a41f20, 0xc42087e4d0, 0x199, 0xc4206d5c80) | |
| /usr/local/go/src/encoding/gob/encode.go:707 +0x258 | |
| encoding/gob.(*Encoder).EncodeValue(0xc4213c6640, 0x1aca2c0, 0xc42087e4d0, 0x16, 0x0, 0x0) | |
| /usr/local/go/src/encoding/gob/encoder.go:247 +0x6eb | |
| encoding/gob.(*Encoder).Encode(0xc4213c6640, 0x1aca2c0, 0xc42087e4d0, 0x0, 0x0) | |
| /usr/local/go/src/encoding/gob/encoder.go:175 +0x73 | |
| github.com/cockroachdb/cockroach/kv.grpcTransportFactory.func1() | |
| /go/src/github.com/cockroachdb/cockroach/kv/transport_race.go:94 +0x39c | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc421a8b1c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1135 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420da0a00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1429 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc420effe00, 0xc4209bdb00, 0xc421302850, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x813 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0xb3 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc4217d59b0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1122 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2778 +0x184a | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc4214481e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1040 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cockroach/util/netutil.MakeServer.func2() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:100 +0x8e | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420b492c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1273 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc420d60d60) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1129 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1011 +0x2f9 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc4214492c0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1286 [IO wait]: | |
| net.runtime_pollWait(0x7fc78fa10f10, 0x72, 0x5f17b3) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x5e | |
| net.(*pollDesc).wait(0xc421473100, 0x72, 0x5ee068, 0x19c4020) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b | |
| net.(*pollDesc).waitRead(0xc421473100, 0x26118c0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x42 | |
| net.(*netFD).Read(0xc4214730a0, 0xc4209cd400, 0x400, 0x400, 0x0, 0x26118c0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1d9 | |
| net.(*conn).Read(0xc42007e2d0, 0xc4209cd400, 0x400, 0x400, 0x102, 0xc4209cd40d, 0xc4209cd405) | |
| /usr/local/go/src/net/net.go:173 +0x97 | |
| net.(*TCPConn).Read(0xc42007e2d0, 0xc4209cd400, 0x400, 0x400, 0xc42094b540, 0xba84a9, 0xc42094b540) | |
| <autogenerated>:69 +0x73 | |
| crypto/tls.(*block).readFromUntil(0xc421450b40, 0x7fc78fa16c70, 0xc42007e2d0, 0x5, 0xc42007e2d0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:472 +0x10d | |
| crypto/tls.(*Conn).readRecord(0xc421667c00, 0x1d7b117, 0xc421667d00, 0xc42094b9a8) | |
| /usr/local/go/src/crypto/tls/conn.go:574 +0x13d | |
| crypto/tls.(*Conn).Read(0xc421667c00, 0xc421592000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1109 +0x186 | |
| bufio.(*Reader).fill(0xc4209e5800) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x197 | |
| bufio.(*Reader).Read(0xc4209e5800, 0xc421025ef8, 0x9, 0x9, 0x0, 0x0, 0xc42094bb48) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x6ac | |
| io.ReadAtLeast(0x2609f40, 0xc4209e5800, 0xc421025ef8, 0x9, 0x9, 0x9, 0xc4211872f8, 0xc4202ed040, 0x0) | |
| /usr/local/go/src/io/io.go:307 +0xb2 | |
| io.ReadFull(0x2609f40, 0xc4209e5800, 0xc421025ef8, 0x9, 0x9, 0x1, 0xc42193f710, 0x0) | |
| /usr/local/go/src/io/io.go:325 +0x73 | |
| golang.org/x/net/http2.readFrameHeader(0xc421025ef8, 0x9, 0x9, 0x2609f40, 0xc4209e5800, 0x0, 0x0, 0xc420ee5770, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0xa3 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc421025ec0, 0x0, 0x0, 0x0, 0x0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0x105 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc420890ba0, 0xc421187a10, 0xc421187a10, 0xc42094bec0, 0x1) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53 | |
| google.golang.org/grpc/transport.(*http2Client).reader(0xc421136500) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:921 +0x154 | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:203 +0xc1a | |
| goroutine 1248 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).Start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:935 +0x19d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc420d600e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1241 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*bookie).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x4c8 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc420fbe7a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1039 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/sql.NewExecutor.func1() | |
| /go/src/github.com/cockroachdb/cockroach/sql/executor.go:288 +0x1c7 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc4200911a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1046 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cockroach/server.(*Server).Start.func6() | |
| /go/src/github.com/cockroachdb/cockroach/server/server.go:409 +0x116 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420b495a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1217 [IO wait]: | |
| net.runtime_pollWait(0x7fc78fa3d610, 0x72, 0x5f17b3) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x5e | |
| net.(*pollDesc).wait(0xc4207e14f0, 0x72, 0x5ee068, 0x19c4020) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b | |
| net.(*pollDesc).waitRead(0xc4207e14f0, 0x26118c0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x42 | |
| net.(*netFD).Read(0xc4207e1490, 0xc4209cdc00, 0x400, 0x400, 0x0, 0x26118c0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_unix.go:243 +0x1d9 | |
| net.(*conn).Read(0xc42007e2c0, 0xc4209cdc00, 0x400, 0x400, 0x63, 0x0, 0x1d7aee8) | |
| /usr/local/go/src/net/net.go:173 +0x97 | |
| net.(*TCPConn).Read(0xc42007e2c0, 0xc4209cdc00, 0x400, 0x400, 0x1, 0x260a4c0, 0xc42007c020) | |
| <autogenerated>:69 +0x73 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42162f080, 0xc4209cdc00, 0x400, 0x400, 0xc4206a3318, 0xba84a9, 0xc4206a3318) | |
| /go/src/github.com/cockroachdb/cmux/cmux.go:218 +0x183 | |
| crypto/tls.(*block).readFromUntil(0xc421450bd0, 0x7fc78f9c9648, 0xc42162f080, 0x5, 0xc42162f080, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:472 +0x10d | |
| crypto/tls.(*Conn).readRecord(0xc420a2a000, 0x1d7b117, 0xc420a2a100, 0x5f3bf4) | |
| /usr/local/go/src/crypto/tls/conn.go:574 +0x13d | |
| crypto/tls.(*Conn).Read(0xc420a2a000, 0xc420f3e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1109 +0x186 | |
| bufio.(*Reader).fill(0xc420c2f140) | |
| /usr/local/go/src/bufio/bufio.go:97 +0x197 | |
| bufio.(*Reader).Read(0xc420c2f140, 0xc42102fbf8, 0x9, 0x9, 0x10, 0x10, 0xc42193f620) | |
| /usr/local/go/src/bufio/bufio.go:209 +0x6ac | |
| io.ReadAtLeast(0x2609f40, 0xc420c2f140, 0xc42102fbf8, 0x9, 0x9, 0x9, 0x61c2ba, 0xc420adeb60, 0xc421187538) | |
| /usr/local/go/src/io/io.go:307 +0xb2 | |
| io.ReadFull(0x2609f40, 0xc420c2f140, 0xc42102fbf8, 0x9, 0x9, 0x0, 0xc4201ed770, 0xc4206a3a00) | |
| /usr/local/go/src/io/io.go:325 +0x73 | |
| golang.org/x/net/http2.readFrameHeader(0xc42102fbf8, 0x9, 0x9, 0x2609f40, 0xc420c2f140, 0x0, 0xc400000000, 0xc421187538, 0x260dec0) | |
| /go/src/golang.org/x/net/http2/frame.go:237 +0xa3 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc42102fbc0, 0x0, 0x260a4c0, 0xc42007c020, 0xc4211876b0) | |
| /go/src/golang.org/x/net/http2/frame.go:464 +0x105 | |
| google.golang.org/grpc/transport.(*framer).readFrame(0xc4208f3a70, 0xc421187680, 0xc421187680, 0xc4206a3cc0, 0x1) | |
| /go/src/google.golang.org/grpc/transport/http_util.go:505 +0x53 | |
| google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc421210000, 0xc4208f3b00) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x30a | |
| google.golang.org/grpc.(*Server).serveStreams(0xc420ddf360, 0x2623e80, 0xc421210000) | |
| /go/src/google.golang.org/grpc/server.go:424 +0x1d6 | |
| google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc420ddf360, 0x2624120, 0xc420a2a000, 0x2610b00, 0xc421171290) | |
| /go/src/google.golang.org/grpc/server.go:411 +0x4f1 | |
| google.golang.org/grpc.(*Server).handleRawConn(0xc420ddf360, 0x2624180, 0xc42162f080) | |
| /go/src/google.golang.org/grpc/server.go:388 +0x5c3 | |
| created by google.golang.org/grpc.(*Server).Serve | |
| /go/src/google.golang.org/grpc/server.go:357 +0x20a | |
| goroutine 1120 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cockroach/server.(*Server).Start.func2() | |
| /go/src/github.com/cockroachdb/cockroach/server/server.go:380 +0xae | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c9880) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1154 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4210c98a0, 0xc42119daa0, 0x1994820, 0x25f2480, 0x1a19140) | |
| <autogenerated>:9 +0xa4 | |
| net/http.(*Server).Serve(0xc420cb6300, 0x261c440, 0xc4210c98a0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2273 +0x249 | |
| github.com/cockroachdb/cockroach/server.(*Server).Start.func4() | |
| /go/src/github.com/cockroachdb/cockroach/server/server.go:396 +0x54 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c9900) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1460 [select, 7 minutes]: | |
| google.golang.org/grpc/transport.(*http2Client).controller(0xc420f83000) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x8c9 | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:233 +0x14bd | |
| goroutine 1714 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*Replica).addWriteCmd(0xc421684c00, 0x7fc78fa10408, 0xc421b0d440, 0x1471b8de7e32f653, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:1457 +0x4d5 | |
| github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc421684c00, 0x7fc78fa10408, 0xc421b0d440, 0x1471b8de7e32f653, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:1001 +0x2a5 | |
| github.com/cockroachdb/cockroach/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica_range_lease.go:125 +0x22d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask.func1(0xc4207dc6c0, 0x230c2b4, 0x1e, 0x8f, 0xc421b0d3e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:234 +0xa3 | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:235 +0xca | |
| goroutine 1052 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*bookie).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x4c8 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420b49920) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1162 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*bookie).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/reservation.go:274 +0x4c8 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c9c80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1266 [select, 6 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLease(0xc421684c00, 0x7fc78fa10408, 0xc42026b860, 0xc420fa9c60) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:721 +0x6bc | |
| github.com/cockroachdb/cockroach/storage.(*Replica).getLeaseForGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:2784 +0xac | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc4207dc6c0, 0xc4206c9d90, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:218 +0x10e | |
| github.com/cockroachdb/cockroach/storage.(*Replica).getLeaseForGossip(0xc421684c00, 0x7fc78fa10408, 0xc42026b860, 0x10, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:2815 +0x218 | |
| github.com/cockroachdb/cockroach/storage.(*Store).maybeGossipSystemConfig(0xc421356780, 0xc4206e3b60, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1080 +0xd2 | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1033 +0x1d3 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4213030f0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1134 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420da09e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1051 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:948 +0x416 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420375100) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1276 [select]: | |
| github.com/cockroachdb/cockroach/util/retry.(*Retry).Next(0xc421a00700, 0x7fc78fa10408) | |
| /go/src/github.com/cockroachdb/cockroach/util/retry/retry.go:128 +0x307 | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk(0xc420a0fdd0, 0x7fc78fa10408, 0xc421b4acf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:683 +0x348 | |
| github.com/cockroachdb/cockroach/kv.(*DistSender).Send(0xc420a0fdd0, 0x7fc78fa10408, 0xc421b4acf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:606 +0x1a1 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc420a09c30, 0x7fc78fa10408, 0xc421b4acf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:391 +0x68a | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).send(0xc42101c780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:495 +0x294 | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).(github.com/cockroachdb/cockroach/internal/client.send)-fm(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:440 +0x7d | |
| github.com/cockroachdb/cockroach/internal/client.sendAndFill(0xc421a01900, 0xc42143c600, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:422 +0x1ac | |
| github.com/cockroachdb/cockroach/internal/client.(*DB).Run(0xc42101c780, 0xc42143c600, 0x1, 0x1) | |
| /go/src/github.com/cockroachdb/cockroach/internal/client/db.go:440 +0xd5 | |
| github.com/cockroachdb/cockroach/storage.(*replicaGCQueue).process(0xc420b72000, 0x7fc78fa10408, 0xc421b4ac90, 0x1471b8dead1c9737, 0x0, 0xc4201ae900, 0xc42100a000, 0x13, 0x20, 0x5f17b3, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica_gc_queue.go:156 +0x2bb | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processReplica(0xc420b72000, 0xc4201ae900, 0xc420c01780, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:489 +0x64c | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1.2() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:428 +0x70 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc4207dc6c0, 0xc421a01e58, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:218 +0x10e | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:432 +0x35a | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc420d60dc0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1437 [select, 7 minutes]: | |
| google.golang.org/grpc.(*addrConn).transportMonitor(0xc420807a40) | |
| /go/src/google.golang.org/grpc/clientconn.go:720 +0x7d7 | |
| google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420807a40) | |
| /go/src/google.golang.org/grpc/clientconn.go:479 +0x26e | |
| created by google.golang.org/grpc.(*ClientConn).resetAddrConn | |
| /go/src/google.golang.org/grpc/clientconn.go:480 +0x647 | |
| goroutine 1097 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/sql.(*SchemaChangeManager).Start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/sql/schema_changer.go:705 +0x1a60 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc4201ba040) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1138 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420da0a80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1155 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4210c98c0, 0xc42119db90, 0xc42119cfc0, 0xc42119cfc0, 0x2613c40) | |
| <autogenerated>:9 +0xa4 | |
| crypto/tls.(*listener).Accept(0xc4210c9920, 0xc42119db30, 0x1994820, 0x25f2480, 0x1a19140) | |
| /usr/local/go/src/crypto/tls/tls.go:52 +0x56 | |
| net/http.(*Server).Serve(0xc420cb6280, 0x2615cc0, 0xc4210c9920, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2273 +0x249 | |
| github.com/cockroachdb/cockroach/server.(*Server).Start.func5() | |
| /go/src/github.com/cockroachdb/cockroach/server/server.go:403 +0x8f | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c9940) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1146 [select, 2 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc42113fe00, 0xc421632f00, 0x0, 0x3b9aca00) | |
| /go/src/google.golang.org/grpc/clientconn.go:683 +0x9a9 | |
| google.golang.org/grpc.(*addrConn).transportMonitor(0xc42113fe00) | |
| /go/src/google.golang.org/grpc/clientconn.go:763 +0x407 | |
| google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc42113fe00) | |
| /go/src/google.golang.org/grpc/clientconn.go:479 +0x26e | |
| created by google.golang.org/grpc.(*ClientConn).resetAddrConn | |
| /go/src/google.golang.org/grpc/clientconn.go:480 +0x647 | |
| goroutine 1117 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/sql.NewExecutor.func1() | |
| /go/src/github.com/cockroachdb/cockroach/sql/executor.go:288 +0x1c7 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc421071440) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1227 [IO wait, 7 minutes]: | |
| net.runtime_pollWait(0x7fc78fa10fd0, 0x72, 0x5f17b3) | |
| /usr/local/go/src/runtime/netpoll.go:160 +0x5e | |
| net.(*pollDesc).wait(0xc421150290, 0x72, 0x5ee068, 0x19c4020) | |
| /usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b | |
| net.(*pollDesc).waitRead(0xc421150290, 0x26118c0, 0xc42007c0b0) | |
| /usr/local/go/src/net/fd_poll_runtime.go:78 +0x42 | |
| net.(*netFD).accept(0xc421150230, 0x0, 0x260ec80, 0xc42116c7c0) | |
| /usr/local/go/src/net/fd_unix.go:419 +0x2b8 | |
| net.(*TCPListener).accept(0xc420026410, 0xc42007e400, 0xc4204e5ec8, 0x61652f) | |
| /usr/local/go/src/net/tcpsock_posix.go:132 +0x51 | |
| net.(*TCPListener).Accept(0xc420026410, 0x1d77fc8, 0xc420c01d80, 0x2624420, 0xc42007e400) | |
| /usr/local/go/src/net/tcpsock.go:222 +0x50 | |
| github.com/cockroachdb/cmux.(*cMux).Serve(0xc420c01d80, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cmux/cmux.go:124 +0xdd | |
| github.com/cockroachdb/cockroach/server.(*Server).Start.func11() | |
| /go/src/github.com/cockroachdb/cockroach/server/server.go:497 +0x47 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4215517a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1041 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cockroach/util/netutil.MakeServer.func2() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:100 +0x8e | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420b49320) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1033 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc420c63b00) | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x64 | |
| created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0xb61 | |
| goroutine 1275 [select, 3 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLease(0xc4201ae900, 0x7fc78fa10408, 0xc420eaa360, 0x15) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:721 +0x6bc | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processReplica(0xc4209ce9c0, 0xc4201ae900, 0xc420c01780, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:477 +0x88c | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1.2() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:428 +0x70 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc4207dc6c0, 0xc4206c7e78, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:218 +0x10e | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:432 +0x35a | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc420d60da0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1274 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc420d60d80) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1283 [select, 7 minutes]: | |
| google.golang.org/grpc/transport.(*http2Server).controller(0xc421210000) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:673 +0xa56 | |
| created by google.golang.org/grpc/transport.newHTTP2Server | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:139 +0xa91 | |
| goroutine 1226 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/sql.(*SchemaChangeManager).Start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/sql/schema_changer.go:705 +0x1a60 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc421551760) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1148 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc420a14580, 0xc421555d00, 0xc420400be0, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x813 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0xb3 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4211ed7a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1223 [select]: | |
| github.com/cockroachdb/cockroach/server.(*Server).startSampleEnvironment.func1() | |
| /go/src/github.com/cockroachdb/cockroach/server/server.go:626 +0x1b4 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc421551720) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1035 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store_pool.go:340 +0x519 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420b48f00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1229 [select, 7 minutes]: | |
| google.golang.org/grpc.(*addrConn).transportMonitor(0xc4203c8500) | |
| /go/src/google.golang.org/grpc/clientconn.go:720 +0x7d7 | |
| google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4203c8500) | |
| /go/src/google.golang.org/grpc/clientconn.go:479 +0x26e | |
| created by google.golang.org/grpc.(*ClientConn).resetAddrConn | |
| /go/src/google.golang.org/grpc/clientconn.go:480 +0x647 | |
| goroutine 1160 [select]: | |
| github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1() | |
| /go/src/github.com/cockroachdb/cockroach/gossip/gossip.go:911 +0x621 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc421402860) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1091 [select]: | |
| github.com/cockroachdb/cockroach/server.(*Node).startComputePeriodicMetrics.func1() | |
| /go/src/github.com/cockroachdb/cockroach/server/node.go:631 +0x2f3 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc4201e25a0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1287 [select, 7 minutes]: | |
| google.golang.org/grpc/transport.(*http2Client).controller(0xc421136500) | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x8c9 | |
| created by google.golang.org/grpc/transport.newHTTP2Client | |
| /go/src/google.golang.org/grpc/transport/http2_client.go:233 +0x14bd | |
| goroutine 1095 [select]: | |
| github.com/cockroachdb/cockroach/ts.(*poller).start.func1() | |
| /go/src/github.com/cockroachdb/cockroach/ts/db.go:81 +0x1c7 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc4206d9b10) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1137 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420da0a40) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1278 [select]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420cb6400, 0xecf60a309, 0xc4311046c6, 0x2962d60, 0xc420c01780, 0xc4207dc6c0, 0xc4201ae900, 0xc4213569b0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x299 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1.1(0xc4201ae900, 0xc42119d140) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:259 +0x124 | |
| github.com/cockroachdb/cockroach/storage.(*storeRangeSet).Visit(0xc42119d200, 0xc421aaf780) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:243 +0x453 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:261 +0x3a5 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc420d60e00) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1058 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc420effe00, 0xc420b4c200, 0xc4203823f0, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x813 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0xb3 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420b904b0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1037 [select]: | |
| github.com/cockroachdb/cockroach/storage.NewRaftTransport.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:172 +0x12f6 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420374950) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1096 [select]: | |
| github.com/cockroachdb/cockroach/server.(*Node).startWriteSummaries.func1() | |
| /go/src/github.com/cockroachdb/cockroach/server/node.go:667 +0x32d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc4201ba020) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1270 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc420d60c40) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1104 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc420effe00, 0xc420b4c700, 0xc420375050, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x813 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0xb3 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420964000) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1119 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cockroach/util/netutil.MakeServer.func2() | |
| /go/src/github.com/cockroachdb/cockroach/util/netutil/net.go:100 +0x8e | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4210c9700) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1233 [select]: | |
| github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc420a14580, 0xc4213ee000, 0xc4214027c0, 0xf, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:286 +0x813 | |
| github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1.1.1() | |
| /go/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0xb3 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4213e10b0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1221 [select]: | |
| github.com/cockroachdb/cockroach/server.(*Node).startGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/server/node.go:591 +0x545 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc42141ce40) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1136 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420da0a20) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1272 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/queue.go:407 +0x395 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc420d60d40) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1071 [chan receive, 7 minutes]: | |
| database/sql.(*DB).connectionOpener(0xc420bf1ad0) | |
| /usr/local/go/src/database/sql/sql.go:730 +0x6e | |
| created by database/sql.Open | |
| /usr/local/go/src/database/sql/sql.go:493 +0x2af | |
| goroutine 1476 [select]: | |
| google.golang.org/grpc/transport.(*http2Server).controller(0xc421480090) | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:673 +0xa56 | |
| created by google.golang.org/grpc/transport.newHTTP2Server | |
| /go/src/google.golang.org/grpc/transport/http2_server.go:139 +0xa91 | |
| goroutine 1114 [select]: | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc420a09c30) | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:208 +0xde7 | |
| github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm() | |
| /go/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:193 +0x42 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc421431d70) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1128 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).Start.func3() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:935 +0x19d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc420da0760) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1127 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42138d6e0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x58 | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xd8 | |
| goroutine 1245 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421040900) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x58 | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xd8 | |
| goroutine 1271 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc420cb6480, 0xecf60a142, 0xc4201bccd2, 0x2962d60, 0xc420c01780, 0xc4207dc6c0, 0x0, 0xc42007c428) | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:201 +0x299 | |
| github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/scanner.go:264 +0x540 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc420d60c60) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1249 [select, 6 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLease(0xc421684c00, 0x7fc78fa10408, 0xc42051f440, 0xc420947938) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:721 +0x6bc | |
| github.com/cockroachdb/cockroach/storage.(*Replica).getLeaseForGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:2784 +0xac | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc4207dc6c0, 0xc420947a78, 0x0, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:218 +0x10e | |
| github.com/cockroachdb/cockroach/storage.(*Replica).getLeaseForGossip(0xc421684c00, 0x7fc78fa10408, 0xc42051f440, 0xc420947b48, 0x10) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:2815 +0x218 | |
| github.com/cockroachdb/cockroach/storage.(*Replica).maybeGossipFirstRange(0xc421684c00, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:2850 +0x295 | |
| github.com/cockroachdb/cockroach/storage.(*Store).maybeGossipFirstRange(0xc421356780, 0xc4206e3b60, 0x0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1057 +0x204 | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1013 +0x1d3 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc4207dc6c0, 0xc4213030e0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1130 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2() | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:1031 +0x2f9 | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc420c4d830, 0xc4214492d0) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:190 +0x8b | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:191 +0x74 | |
| goroutine 1247 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc421040900) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x58 | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xd8 | |
| goroutine 1125 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42138d6e0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x58 | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xd8 | |
| goroutine 1072 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc4206e3c80) | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:388 +0x64 | |
| created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open | |
| /go/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:389 +0xb61 | |
| goroutine 1124 [chan receive]: | |
| github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1.1(0xc42138d6e0) | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2690 +0x58 | |
| created by github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1 | |
| /go/src/github.com/cockroachdb/cockroach/storage/store.go:2693 +0xd8 | |
| goroutine 1692 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/storage.(*Replica).addWriteCmd(0xc4219f5200, 0x7fc78fa10408, 0xc421b777a0, 0x1471b8de5964ed66, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:1457 +0x4d5 | |
| github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc4219f5200, 0x7fc78fa10408, 0xc421b777a0, 0x1471b8de5964ed66, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, ...) | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica.go:1001 +0x2a5 | |
| github.com/cockroachdb/cockroach/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1() | |
| /go/src/github.com/cockroachdb/cockroach/storage/replica_range_lease.go:125 +0x22d | |
| github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask.func1(0xc420c4d830, 0x230c2b4, 0x1e, 0x8f, 0xc421b77710) | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:234 +0xa3 | |
| created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask | |
| /go/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:235 +0xca | |
| FAIL github.com/cockroachdb/cockroach/testutils/testcluster 480.149s |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment