Created
February 12, 2015 01:37
-
-
Save apple-corps/61b6df698e48386f12f8 to your computer and use it in GitHub Desktop.
Hadoop 2.3.0-cdh5.0.1 balancer stack trace
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| sudo -u hdfs hdfs balancer | |
| 15/02/11 18:33:03 INFO balancer.Balancer: namenodes = [hdfs://whqa/, hdfs://whqa] | |
| 15/02/11 18:33:03 INFO balancer.Balancer: p = Balancer.Parameters[BalancingPolicy.Node, threshold=10.0] | |
| Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved | |
| 15/02/11 18:33:04 INFO net.NetworkTopology: Adding a new node: /9/10.51.28.200:50010 | |
| 15/02/11 18:33:04 INFO net.NetworkTopology: Adding a new node: /7/10.51.28.201:50010 | |
| 15/02/11 18:33:04 INFO net.NetworkTopology: Adding a new node: /9/10.51.28.203:50010 | |
| 15/02/11 18:33:04 INFO net.NetworkTopology: Adding a new node: /8/10.51.28.202:50010 | |
| 15/02/11 18:33:04 INFO balancer.Balancer: 0 over-utilized: [] | |
| 15/02/11 18:33:04 INFO balancer.Balancer: 0 underutilized: [] | |
| The cluster is balanced. Exiting... | |
| 15/02/11 18:33:04 INFO net.NetworkTopology: Adding a new node: /8/10.51.28.202:50010 | |
| 15/02/11 18:33:04 INFO net.NetworkTopology: Adding a new node: /9/10.51.28.203:50010 | |
| 15/02/11 18:33:04 INFO net.NetworkTopology: Adding a new node: /9/10.51.28.200:50010 | |
| 15/02/11 18:33:04 INFO net.NetworkTopology: Adding a new node: /7/10.51.28.201:50010 | |
| 15/02/11 18:33:04 INFO balancer.Balancer: 0 over-utilized: [] | |
| 15/02/11 18:33:04 INFO balancer.Balancer: 0 underutilized: [] | |
| The cluster is balanced. Exiting... | |
| 15/02/11 18:33:04 WARN hdfs.DFSClient: DataStreamer Exception | |
| java.io.FileNotFoundException: ID mismatch. Request id and saved id: 10917162 , 10917163 | |
| at org.apache.hadoop.hdfs.server.namenode.INodeId.checkId(INodeId.java:53) | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:2955) | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:2757) | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2665) | |
| at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:569) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:440) | |
| at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:585) | |
| at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1026) | |
| at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1986) | |
| at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1982) | |
| at java.security.AccessController.doPrivileged(Native Method) | |
| at javax.security.auth.Subject.doAs(Subject.java:415) | |
| at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548) | |
| at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1980) | |
| at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) | |
| at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) | |
| at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) | |
| at java.lang.reflect.Constructor.newInstance(Constructor.java:526) | |
| at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106) | |
| at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:73) | |
| at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1440) | |
| at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1260) | |
| at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:525) | |
| Caused by: org.apache.hadoop.ipc.RemoteException(java.io.FileNotFoundException): ID mismatch. Request id and saved id: 10917162 , 10917163 | |
| at org.apache.hadoop.hdfs.server.namenode.INodeId.checkId(INodeId.java:53) | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:2955) | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:2757) | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2665) | |
| at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:569) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:440) | |
| at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:585) | |
| at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1026) | |
| at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1986) | |
| at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1982) | |
| at java.security.AccessController.doPrivileged(Native Method) | |
| at javax.security.auth.Subject.doAs(Subject.java:415) | |
| at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548) | |
| at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1980) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1362) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206) | |
| at com.sun.proxy.$Proxy11.addBlock(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:361) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:606) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:186) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) | |
| at com.sun.proxy.$Proxy12.addBlock(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1437) | |
| ... 2 more | |
| 15/02/11 18:33:04 WARN hdfs.DFSClient: DataStreamer Exception | |
| org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /system/balancer.id: File does not exist. Holder DFSClient_NONMAPREDUCE_-46205058_1 does not have any open files. | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:2937) | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:2757) | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2665) | |
| at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:569) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:440) | |
| at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:585) | |
| at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1026) | |
| at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1986) | |
| at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1982) | |
| at java.security.AccessController.doPrivileged(Native Method) | |
| at javax.security.auth.Subject.doAs(Subject.java:415) | |
| at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548) | |
| at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1980) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1362) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206) | |
| at com.sun.proxy.$Proxy11.addBlock(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:361) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:606) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:186) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) | |
| at com.sun.proxy.$Proxy12.addBlock(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1437) | |
| at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1260) | |
| at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:525) | |
| Balancing took 1.719 seconds | |
| 15/02/11 18:33:04 ERROR hdfs.DFSClient: Failed to close file /system/balancer.id | |
| org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /system/balancer.id: File does not exist. Holder DFSClient_NONMAPREDUCE_-46205058_1 does not have any open files. | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:2937) | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:2757) | |
| at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2665) | |
| at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:569) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:440) | |
| at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:585) | |
| at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1026) | |
| at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1986) | |
| at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1982) | |
| at java.security.AccessController.doPrivileged(Native Method) | |
| at javax.security.auth.Subject.doAs(Subject.java:415) | |
| at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548) | |
| at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1980) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1362) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206) | |
| at com.sun.proxy.$Proxy11.addBlock(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:361) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:606) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:186) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) | |
| at com.sun.proxy.$Proxy12.addBlock(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1437) | |
| at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1260) | |
| at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:525) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment