Created
December 17, 2012 01:56
-
-
Save ericacm/4315253 to your computer and use it in GitHub Desktop.
Test after building from source. I also removed the RetryLoop. This is what happens after I kill the ZK leader. The original leader localhost:14181 loses leadership and no member gets hasLeadership though the participant.isLeader is true for localhost:14189.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| This is localhost:14181 - the original leader | |
| 20:45:43.175 [RMI TCP Connection(2)-127.0.0.1-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14181 hasLeadership=true, Cluster participants: localhost:14181, localhost:14189, leader=localhost:14181 | |
| 20:46:00.017 [ThreadPoolTaskScheduler-1] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (16724) | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:247) [ZookeeperServiceImpl.class:na] | |
| at com.nyx.services.notification.DeliveryServiceImpl.scheduledMessages(DeliveryService.scala:280) [DeliveryServiceImpl.class:na] | |
| at com.nyx.services.notification.DeliveryServiceImpl$$anonfun$start$6.apply$mcV$sp(DeliveryService.scala:225) [DeliveryServiceImpl$$anonfun$start$6.class:na] | |
| at com.nyx.services.SchedulerServiceImpl$$anon$1.run(SchedulerService.scala:134) [SchedulerServiceImpl$$anon$1.class:na] | |
| at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:51) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE] | |
| at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE] | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439) [na:1.6.0_37] | |
| at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) [na:1.6.0_37] | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:138) [na:1.6.0_37] | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:98) [na:1.6.0_37] | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:206) [na:1.6.0_37] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [na:1.6.0_37] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [na:1.6.0_37] | |
| at java.lang.Thread.run(Thread.java:680) [na:1.6.0_37] | |
| 20:46:13.319 [RMI TCP Connection(2)-127.0.0.1-EventThread] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (30027) | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:247) [ZookeeperServiceImpl.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) [ZookeeperServiceImpl$$anon$2.class:na] | |
| at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) [curator-framework-1.2.3.jar:na] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| 20:46:13.320 [RMI TCP Connection(2)-127.0.0.1-EventThread] ERROR c.n.c.f.imps.CuratorFrameworkImpl - Watcher exception | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) ~[curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) ~[curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) ~[curator-recipes-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:247) ~[ZookeeperServiceImpl.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) ~[ZookeeperServiceImpl$$anon$2.class:na] | |
| at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) ~[curator-framework-1.2.3.jar:na] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| 20:46:13.321 [RMI TCP Connection(2)-127.0.0.1-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=None state=SyncConnected | |
| 20:46:13.342 [RMI TCP Connection(2)-127.0.0.1-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14181 hasLeadership=false, Cluster participants: localhost:14181, localhost:14189, leader=localhost:14189 | |
| 20:46:13.342 [RMI TCP Connection(2)-127.0.0.1-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=NodeChildrenChanged state=SyncConnected | |
| 20:46:13.351 [RMI TCP Connection(2)-127.0.0.1-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14181 hasLeadership=false, Cluster participants: localhost:14189, localhost:14181, leader=localhost:14189 | |
| This is localhost:14189 | |
| 20:45:48.870 [main-EventThread] ERROR c.n.c.f.recipes.leader.LeaderLatch - An error ocurred checking the leadership. | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /ls | |
| at org.apache.zookeeper.KeeperException.create(KeeperException.java:99) ~[zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) ~[zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1581) ~[zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) ~[curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.checkForLeadership(LeaderLatch.java:325) ~[curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.access$200(LeaderLatch.java:49) ~[curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch$3.process(LeaderLatch.java:349) ~[curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:50) [curator-framework-1.2.3.jar:na] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| 20:45:48.871 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=NodeChildrenChanged state=SyncConnected | |
| 20:45:48.980 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14189 hasLeadership=false, Cluster participants: localhost:14181, localhost:14189, leader=localhost:14181 | |
| 20:46:13.334 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=NodeChildrenChanged state=SyncConnected | |
| 20:46:13.342 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14189 hasLeadership=false, Cluster participants: localhost:14189, localhost:14181, leader=localhost:14189 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment