Created
November 12, 2015 21:57
-
-
Save mahdibh/76b230e25a3f7113349e to your computer and use it in GitHub Desktop.
Kafka/zookeeper shutdown issue
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-11-12T12:03:35.198Z INFO [d(ip-10-0-0-1.ec2.internal:2181)] [org.apache.zookeeper.ClientCnxn ]: Opening socket connection to server ip-10-0-0-1.ec2.internal/10.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) | |
| 2015-11-12T12:03:35.479Z INFO [d(ip-10-0-0-1.ec2.internal:2181)] [org.apache.zookeeper.ClientCnxn ]: Client session timed out, have not heard from server in 858ms for sessionid 0x1250c5c0f1f5001c, closing socket connection and attempting reconnect | |
| 2015-11-12T12:03:37.398Z INFO [ad(ip-10.0.0.2.ec2.internal:2181)] [org.apache.zookeeper.ClientCnxn ]: Opening socket connection to server ip-10.0.0.2.ec2.internal/10.0.0.2:2181. Will not attempt to authenticate using SASL (unknown error) | |
| 2015-11-12T12:03:37.787Z INFO [main-EventThread ] [org.I0Itec.zkclient.ZkClient ]: zookeeper state changed (Disconnected) | |
| 2015-11-12T12:03:38.256Z INFO [ad(ip-10.0.0.2.ec2.internal:2181)] [org.apache.zookeeper.ClientCnxn ]: Client session timed out, have not heard from server in 2677ms for sessionid 0x1250c5c0f1f5001c, closing socket connection and attempting reconnect | |
| 2015-11-12T12:03:38.549Z INFO [d(ip-10.0.0.3.ec2.internal:2181)] [org.apache.zookeeper.ClientCnxn ]: Opening socket connection to server ip-10.0.0.3.ec2.internal/1 | |
| 0.0.36.107:2181. Will not attempt to authenticate using SASL (unknown error) | |
| 2015-11-12T12:03:38.551Z INFO [d(ip-10.0.0.3.ec2.internal:2181)] [org.apache.zookeeper.ClientCnxn ]: Socket connection established to ip-10.0.0.3.ec2.internal/10.0.0.2:2181, initiating session | |
| 2015-11-12T12:03:38.553Z INFO [main-EventThread ] [org.I0Itec.zkclient.ZkClient ]: zookeeper state changed (Expired) | |
| 2015-11-12T12:03:38.553Z INFO [main-EventThread ] [org.apache.zookeeper.ZooKeeper ]: Initiating client connection, connectString=zookeeper21.example.com:2181,zookeeper19.example.com:2181,zookeeper22.example.com:2181,zookeeper18.example.com:2181,zookeeper20.example.com:2181,zookeeper16.example.com:2181,zookeeper15.example.com:2181/foo/kafka/central sessionTimeout=6000 watcher=org.I0Itec.zkclient.ZkClient@3bbc39f8 | |
| 2015-11-12T12:03:39.949Z INFO [d(ip-10.0.0.3.ec2.internal:2181)] [org.apache.zookeeper.ClientCnxn ]: Unable to reconnect to ZooKeeper service, session 0x1250c5c0f1f5001c has expired, closing socket connection | |
| 2015-11-12T12:03:40.063Z INFO [main-EventThread ] [org.I0Itec.zkclient.ZkClient ]: Unable to re-establish connection. Notifying consumer of the following exception: | |
| org.I0Itec.zkclient.exception.ZkException: Unable to connect to zookeeper21.example.com:2181,zookeeper19.example.com:2181,zookeeper22.example.com:2181,zookeeper18.example.com:2181,zookeeper20.example.com:2181,zookeeper16.example.com:2181,zookeeper15.example.com:2181/foo/kafka/central | |
| at org.I0Itec.zkclient.ZkConnection.connect(ZkConnection.java:69) | |
| at org.I0Itec.zkclient.ZkClient.reconnect(ZkClient.java:1176) | |
| at org.I0Itec.zkclient.ZkClient.processStateChanged(ZkClient.java:649) | |
| at org.I0Itec.zkclient.ZkClient.process(ZkClient.java:560) | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522) | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498) | |
| Caused by: java.net.UnknownHostException: zookeeper16.example.com: unknown error | |
| at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method) | |
| at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928) | |
| at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323) | |
| at java.net.InetAddress.getAllByName0(InetAddress.java:1276) | |
| at java.net.InetAddress.getAllByName(InetAddress.java:1192) | |
| at java.net.InetAddress.getAllByName(InetAddress.java:1126) | |
| at org.apache.zookeeper.client.StaticHostProvider.<init>(StaticHostProvider.java:61) | |
| at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:445) | |
| at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380) | |
| at org.I0Itec.zkclient.ZkConnection.connect(ZkConnection.java:67) | |
| ... 5 more | |
| 2015-11-12T12:03:40.068Z INFO [main-EventThread ] [org.apache.zookeeper.ClientCnxn ]: EventThread shut down | |
| 2015-11-12T12:03:40.087Z FATAL [example.com:2181/foo/kafka/central] [kafka.server.KafkaHealthcheck ]: Could not establish session with zookeeper | |
| org.I0Itec.zkclient.exception.ZkException: Unable to connect to zookeeper21.example.com:2181,zookeeper19.example.com:2181,zookeeper22.example.com:2181,zookeeper18.example.com:2181,zookeeper20.example.com:2181,zookeeper16.example.com:2181,zookeeper15.example.com:2181/foo/kafka/central | |
| at org.I0Itec.zkclient.ZkConnection.connect(ZkConnection.java:69) | |
| at org.I0Itec.zkclient.ZkClient.reconnect(ZkClient.java:1176) | |
| at org.I0Itec.zkclient.ZkClient.processStateChanged(ZkClient.java:649) | |
| at org.I0Itec.zkclient.ZkClient.process(ZkClient.java:560) | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522) | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498) | |
| Caused by: java.net.UnknownHostException: zookeeper16.example.com: unknown error | |
| at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method) | |
| at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928) | |
| at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323) | |
| at java.net.InetAddress.getAllByName0(InetAddress.java:1276) | |
| at java.net.InetAddress.getAllByName(InetAddress.java:1192) | |
| at java.net.InetAddress.getAllByName(InetAddress.java:1126) | |
| at org.apache.zookeeper.client.StaticHostProvider.<init>(StaticHostProvider.java:61) | |
| at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:445) | |
| at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380) | |
| at org.I0Itec.zkclient.ZkConnection.connect(ZkConnection.java:67) | |
| ... 5 more | |
| 2015-11-12T12:03:40.098Z INFO [Thread-2 ] [kafka.server.KafkaServer ]: [Kafka Server 13], shutting down | |
| 2015-11-12T12:03:40.099Z INFO [Thread-2 ] [kafka.server.KafkaServer ]: [Kafka Server 13], Starting controlled shutdown | |
| 2015-11-12T12:03:40.101Z WARN [Thread-2 ] [kafka.utils.Utils$ ]: | |
| java.lang.NullPointerException | |
| at org.I0Itec.zkclient.ZkConnection.readData(ZkConnection.java:117) | |
| at org.I0Itec.zkclient.ZkClient$10.call(ZkClient.java:992) | |
| at org.I0Itec.zkclient.ZkClient$10.call(ZkClient.java:988) | |
| at org.I0Itec.zkclient.ZkClient.retryUntilConnected(ZkClient.java:883) | |
| at org.I0Itec.zkclient.ZkClient.readData(ZkClient.java:988) | |
| at org.I0Itec.zkclient.ZkClient.readData(ZkClient.java:983) | |
| at kafka.utils.ZkUtils$.readDataMaybeNull(ZkUtils.scala:450) | |
| at kafka.utils.ZkUtils$.getController(ZkUtils.scala:65) | |
| at kafka.server.KafkaServer.kafka$server$KafkaServer$$controlledShutdown(KafkaServer.scala:194) | |
| at kafka.server.KafkaServer$$anonfun$shutdown$1.apply$mcV$sp(KafkaServer.scala:269) | |
| at kafka.utils.Utils$.swallow(Utils.scala:172) | |
| at kafka.utils.Logging$class.swallowWarn(Logging.scala:92) | |
| at kafka.utils.Utils$.swallowWarn(Utils.scala:45) | |
| at kafka.utils.Logging$class.swallow(Logging.scala:94) | |
| at kafka.utils.Utils$.swallow(Utils.scala:45) | |
| at kafka.server.KafkaServer.shutdown(KafkaServer.scala:269) | |
| at kafka.server.KafkaServerStartable.shutdown(KafkaServerStartable.scala:42) | |
| at kafka.Kafka$$anon$1.run(Kafka.scala:42) | |
| 2015-11-12T12:03:40.106Z INFO [Thread-2 ] [kafka.network.SocketServer ]: [Socket Server on Broker 13], Shutting down | |
| 2015-11-12T12:05:18.017Z INFO [ReplicaFetcherThread-3-9 ] [kafka.log.Log ]: Rolled new log segment for 'topic-a-1' in 0 ms. | |
| 2015-11-12T12:06:12.864Z INFO [ReplicaFetcherThread-0-12 ] [kafka.log.Log ]: Rolled new log segment for 'topic-b-4' in 0 ms. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment