Last active
August 29, 2015 14:20
-
-
Save haridsv/da7146dce4192957d02d to your computer and use it in GitHub Desktop.
OpenTSDB errors under load
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-27 13:27:17,030 ERROR [New I/O worker #67] RegionClient: Uncaught error during de-serialization of MultiAction(batch=[PutRequest(table="tsdb", key=[0, 0, 22, 85, 27, -35, -64, 0, 0, 1, 0, 0, -64], family="t", qualifiers=[[17, -121]], values=[[0, 0, 0, 10, -64, 92, -47, 108]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=2, region=RegionInfo(table="tsdb", region_name="tsdb,,1430141025970.93e30e6cb050703b91b3aae198c79e3c.", stop_key=[0, 0, 78, 85, 27, -105, 112, 0, 0, 1, 0, 0, -33, 0, 0, 13, 0, 0, 111, 0, 0, 14, 0, 0, 121, 0, 0, 15, 0, 0, 103, 0, 0, 16, 0, 0, 87])), PutRequest(table="tsdb", key=[0, 0, 29, 85, 27, -35, -64, 0, 0, 1, 0, 0, -64], family="t", qualifiers=[[17, -125]], values=[[0, 5, 18, 11]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=2, region=RegionInfo(table="tsdb", region_name="tsdb,,1430141025970.93e30e6cb050703b91b3aae198c79e3c.", stop_key=[0, 0, 78, 85, 27, -105, 112, 0, 0, 1, 0, 0, -33, 0, 0, 13, 0, 0, 111, 0, 0, 14, 0, 0, 121, 0, 0, 15, 0, 0, 103, 0, 0, 16, 0, 0, 87])), PutRequest(table="tsdb", key=[0, 0, 33, 85, 27, -35, -64, 0, 0, 1, 0, 0, -64], family="t", qualifiers=[[17, -125]], values=["\x00\x06q""], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=2, region=RegionInfo(table="tsdb", region_name="tsdb,,1430141025970.93e30e6cb050703b91b3aae198c79e3c.", stop_key=[0, 0, 78, 85, 27, -105, 112, 0, 0, 1, 0, 0, -33, 0, 0, 13, 0, 0, 111, 0, 0, 14, 0, 0, 121, 0, 0, 15, 0, 0, 103, 0, 0, 16, 0, 0, 87])), ... 1020 RPCs not shown ..., PutRequest(table="tsdb", key=[0, 0, 31, 85, 28, 36, 16, 0, 0, 1, 0, 0, 33, 0, 0, 6, 0, 0, 40], family="t", qualifiers=[":\xC1"], values=[[1, -8]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=2, region=RegionInfo(table="tsdb", region_name="tsdb,,1430141025970.93e30e6cb050703b91b3aae198c79e3c.", stop_key=[0, 0, 78, 85, 27, -105, 112, 0, 0, 1, 0, 0, -33, 0, 0, 13, 0, 0, 111, 0, 0, 14, 0, 0, 121, 0, 0, 15, 0, 0, 103, 0, 0, 16, 0, 0, 87]))]), rpcid=12874 | |
| 2015-04-27 13:27:17,031 ERROR [New I/O worker #67] RegionClient: Unexpected exception from downstream on [id: 0x88d11ac2, /10.230.229.36:50555 => /10.230.229.49:60020] | |
| java.lang.IndexOutOfBoundsException: Index: 1, Size: 1 | |
| at java.util.ArrayList.rangeCheck(ArrayList.java:635) ~[na:1.7.0_76] | |
| at java.util.ArrayList.get(ArrayList.java:411) ~[na:1.7.0_76] | |
| at java.util.Collections$UnmodifiableList.get(Collections.java:1211) ~[na:1.7.0_76] | |
| at org.hbase.async.generated.ClientPB$MultiResponse.getRegionActionResult(ClientPB.java:18976) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.MultiAction.deserialize(MultiAction.java:536) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.RegionClient.decode(RegionClient.java:1334) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.RegionClient.decode(RegionClient.java:88) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) ~[netty-3.9.1.Final.jar:na] | |
| at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1093) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) ~[netty-3.9.1.Final.jar:na] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76] | |
| at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-27 13:22:17,746 ERROR [New I/O worker #57] RegionClient: Unexpected exception from downstream on [id: 0x798e3376, /10.230.229.36:50541 => /10.230.229.49:60020] | |
| org.hbase.async.InvalidResponseException: Expected result #566 but got result #567 | |
| at org.hbase.async.MultiAction.deserialize(MultiAction.java:568) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.RegionClient.decode(RegionClient.java:1334) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.RegionClient.decode(RegionClient.java:88) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) ~[netty-3.9.1.Final.jar:na] | |
| at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1093) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) ~[netty-3.9.1.Final.jar:na] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76] | |
| at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-27 13:19:55,579 ERROR [New I/O worker #52] RegionClient: Unexpected exception from downstream on [id: 0xe1ed4d0f, /10.230.229.36:50536 :> /10.230.229.49:60020] | |
| java.nio.channels.ClosedChannelException: null | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.cleanUpWriteBuffer(AbstractNioWorker.java:405) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.close(AbstractNioWorker.java:373) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioClientSocketPipelineSink.eventSunk(NioClientSocketPipelineSink.java:58) ~[netty-3.9.1.Final.jar:na] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendDownstream(HBaseClient.java:2645) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.channel.Channels.close(Channels.java:812) ~[netty-3.9.1.Final.jar:na] | |
| at org.hbase.async.RegionClient.exceptionCaught(RegionClient.java:1109) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1093) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.channel.Channels.fireExceptionCaught(Channels.java:525) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:48) ~[netty-3.9.1.Final.jar:na] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) ~[asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) ~[netty-3.9.1.Final.jar:na] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76] | |
| at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-27 13:27:17,038 WARN [4,queue=4,port=60020] ipc.RpcServer - RpcServer.respondercallId: 12873 service: ClientService methodName: Multi size: 58.8 K connection: 10.230.229.36:50555: output error | |
| 2015-04-27 13:27:17,039 WARN [4,queue=4,port=60020] ipc.RpcServer - B.DefaultRpcServer.handler=34,queue=4,port=60020: caught a ClosedChannelException, this means that the server isthbase01-dnds3-3-crd.eng.sfdc.net/10.230.229.49:60020 was processing a request but the client went away. The error message was: null | |
| 2015-04-27 13:27:17,064 WARN [.reader=8,port=60020] ipc.RpcServer - RpcServer.listener,port=60020: count of bytes read: 0 | |
| java.io.IOException: Connection reset by peer | |
| at sun.nio.ch.FileDispatcherImpl.read0(Native Method) | |
| at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) | |
| at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) | |
| at sun.nio.ch.IOUtil.read(IOUtil.java:197) | |
| at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) | |
| at org.apache.hadoop.hbase.ipc.RpcServer.channelRead(RpcServer.java:2246) | |
| at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1423) | |
| at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:798) | |
| at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:589) | |
| at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:564) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:745) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-27 13:32:17,556 WARN [New I/O worker #85] HBaseClient: Probe Exists(table="tsdb", key=[0, 0, 79, 85, 30, 58, 112, 0, 0, 1, 0, 0, 33], family=null, qualifiers=null, attempt=0, region=RegionInfo(table="tsdb", region_name="tsdb,\x00\x00NU\x1B\x97p\x00\x00\x01\x00\x00\xDF\x00\x00\x0D\x00\x00o\x00\x00\x0E\x00\x00y\x00\x00\x0F\x00\x00g\x00\x00\x10\x00\x00W,1430141397269.0a544d28022eaaaabfee319f3a77f582.", stop_key=[0, 0, -54, 85, 28, -108, -112, 0, 0, 1, 0, 0, 1, 0, 0, 8, 0, 0, 54])) failed | |
| org.hbase.async.RemoteException: org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=isthbase01-dnds4-3-crd.eng.sfdc.net port=60020 startCode=1430135168621. As of locationSeqNum=402070. | |
| at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2787) | |
| at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362) | |
| at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2928) | |
| at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30804) | |
| at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029) | |
| at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107) | |
| at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) | |
| at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) | |
| at java.lang.Thread.run(Thread.java:745) | |
| at org.hbase.async.RegionClient.makeException(RegionClient.java:1473) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.RegionClient.decodeException(RegionClient.java:1491) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.RegionClient.decode(RegionClient.java:1322) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.RegionClient.decode(RegionClient.java:88) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [netty-3.9.1.Final.jar:na] | |
| at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1093) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) [netty-3.9.1.Final.jar:na] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) [netty-3.9.1.Final.jar:na] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76] | |
| at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-27 13:23:55,649 ERROR [New I/O worker #70] RegionClient: After decoding the last message on [id: 0x92afc6d3, /10.230.229.36:35772 :> /10.230.229.44:60020], there was still some undecoded bytes in the channel's buffer (which are going to be lost): ReplayingDecoderBuffer(ridx=60479, widx=61184)="\x00\x00\x03\xFD\xFB\x07\x08\xD0\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD1\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD2\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD3\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD5\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD4\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD6\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD7\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD8\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xD9\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDA\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDB\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDC\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDD\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDE\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xDF\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE0\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE1\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE2\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE3\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE4\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE5\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE6\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE8\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE7\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xE9\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xEB\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xEA\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xEC\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xED\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xEE\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xEF\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF0\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF1\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF3\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF2\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF4\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF5\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF6\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF7\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF8\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xF9\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFA\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFB\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFC\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFD\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFE\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\xFF\x06\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x80\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x82\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x81\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x83\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x84\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x85\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x87\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x86\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x88\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x89\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x8A\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xFD\xFB\x07\x08\x8C\x07\x12\xF5\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\xBD\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,\x00\x01\x10U\x1BQ \x00\x00\x01\x00\x00\x0B,1430140789452.0a02146da97629a03c0032c5141807aa. is not online on isthbase01-dnds2-3-crd.eng.sfdc.net,60020,1430135196555\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-27 13:19:56,245 ERROR [New I/O worker #53] RegionClient: After decoding the last message on [id: 0x9d437a85, /10.230.229.36:50537 :> /10.230.229.49:60020], there was still some undecoded bytes in the channel's buffer (which are going to be lost): ReplayingDecoderBuffer(ridx=12770, widx=13318)="\x00\x00\x03\xD2\xD0\x07\x08\xB9\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xBB\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xBC\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xBD\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xBE\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xBF\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC0\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC1\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC2\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC3\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC4\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC5\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC6\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30806)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)\n\tat java.lang.Thread.run(Thread.java:745)\n(\x00\x00\x00\x03\xD2\xD0\x07\x08\xC7\n\x12\xCA\x07\n1org.apache.hadoop.hbase.NotServingRegionException\x12\x92\x07org.apache.hadoop.hbase.NotServingRegionException: Region tsdb,,1430135589082.f3d3db1f6318e69e8f79e03e8ebcae60. is not online on isthbase01-dnds3-3-crd.eng.sfdc.net,60020,1430135169008\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2796)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.mutate(HRegionServer.java:2995)\n\tat org.apache.hadoop" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-27 13:19:56,245 WARN [RpcServer.responder] ipc.RpcServer - RpcServer.respondercallId: 1771 service: ClientService methodName: Mutate size: 142 connection: 10.230.229.36:50537: output error | |
| 2015-04-27 13:19:56,245 INFO [RpcServer.responder] ipc.RpcServer - RpcServer.responder: asyncWrite | |
| java.io.IOException: Connection reset by peer | |
| at sun.nio.ch.FileDispatcherImpl.writev0(Native Method) | |
| at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:148) | |
| at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:524) | |
| at org.apache.hadoop.hbase.ipc.BufferChain.write(BufferChain.java:106) | |
| at org.apache.hadoop.hbase.ipc.RpcServer.channelWrite(RpcServer.java:2226) | |
| at org.apache.hadoop.hbase.ipc.RpcServer$Responder.processResponse(RpcServer.java:1012) | |
| at org.apache.hadoop.hbase.ipc.RpcServer$Responder.doAsyncWrite(RpcServer.java:952) | |
| at org.apache.hadoop.hbase.ipc.RpcServer$Responder.doRunLoop(RpcServer.java:881) | |
| at org.apache.hadoop.hbase.ipc.RpcServer$Responder.run(RpcServer.java:857) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-27 13:27:17,018 WARN [New I/O worker #67] HBaseClient: Probe Exists(table="tsdb", key=[0, 0, 22, 85, 27, -35, -64, 0, 0, 1, 0, 0, -64], family=null, qualifiers=null, attempt=0, region=RegionInfo(table="tsdb", region_name="tsdb,,1430141025970.93e30e6cb050703b91b3a | |
| ae198c79e3c.", stop_key=[0, 0, 78, 85, 27, -105, 112, 0, 0, 1, 0, 0, -33, 0, 0, 13, 0, 0, 111, 0, 0, 14, 0, 0, 121, 0, 0, 15, 0, 0, 103, 0, 0, 16, 0, 0, 87])) failed | |
| org.hbase.async.RemoteException: org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=isthbase01-dnds4-2-crd.eng.sfdc.net port=60020 startCode=1430135168598. As of locationSeqNum=305417. | |
| at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2787) | |
| at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4362) | |
| at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2928) | |
| at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30804) | |
| at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2029) | |
| at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107) | |
| at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) | |
| at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) | |
| at java.lang.Thread.run(Thread.java:745) | |
| at org.hbase.async.RegionClient.makeException(RegionClient.java:1473) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.RegionClient.decodeException(RegionClient.java:1491) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.RegionClient.decode(RegionClient.java:1322) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.hbase.async.RegionClient.decode(RegionClient.java:88) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) [netty-3.9.1.Final.jar:na] | |
| at org.hbase.async.RegionClient.handleUpstream(RegionClient.java:1093) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) [netty-3.9.1.Final.jar:na] | |
| at org.hbase.async.HBaseClient$RegionClientPipeline.sendUpstream(HBaseClient.java:2655) [asynchbase-1.5.0-sfdc-1.0.0.jar:1.5.0-sfdc-1.0.0] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) [netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) [netty-3.9.1.Final.jar:na] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76] | |
| at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-28 05:50:40,714 ERROR [New I/O worker #33] ConnectionManager: Unexpected exception from downstream for [id: 0x4a556bde, /10.230.229.41:50696 => /10.230.229.36:4242] | |
| java.io.IOException: Broken pipe | |
| at sun.nio.ch.FileChannelImpl.transferTo0(Native Method) ~[na:1.7.0_76] | |
| at sun.nio.ch.FileChannelImpl.transferToDirectly(FileChannelImpl.java:433) ~[na:1.7.0_76] | |
| at sun.nio.ch.FileChannelImpl.transferTo(FileChannelImpl.java:565) ~[na:1.7.0_76] | |
| at org.jboss.netty.channel.DefaultFileRegion.transferTo(DefaultFileRegion.java:68) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.SocketSendBufferPool$FileSendBuffer.transferTo(SocketSendBufferPool.java:331) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.write0(AbstractNioWorker.java:201) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.writeFromUserCode(AbstractNioWorker.java:146) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.handleAcceptedSocket(NioServerSocketPipelineSink.java:99) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.eventSunk(NioServerSocketPipelineSink.java:36) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:60) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.write(Channels.java:704) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.write(Channels.java:671) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:248) ~[netty-3.9.1.Final.jar:na] | |
| at net.opentsdb.tsd.HttpQuery.sendFile(HttpQuery.java:949) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4] | |
| at net.opentsdb.tsd.HttpQuery.sendFile(HttpQuery.java:894) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4] | |
| at net.opentsdb.tsd.GraphHandler.respondAsciiQuery(GraphHandler.java:818) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4] | |
| at net.opentsdb.tsd.GraphHandler.doGraph(GraphHandler.java:214) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4] | |
| at net.opentsdb.tsd.GraphHandler.execute(GraphHandler.java:121) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4] | |
| at net.opentsdb.tsd.RpcHandler.handleHttpQuery(RpcHandler.java:255) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4] | |
| at net.opentsdb.tsd.RpcHandler.messageReceived(RpcHandler.java:163) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:459) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:536) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler.handleUpstream(IdleStateAwareChannelHandler.java:36) ~[netty-3.9.1.Final.jar:na] | |
| at net.opentsdb.tsd.ConnectionManager.handleUpstream(ConnectionManager.java:90) ~[opentsdb-2.0.0-sfdc-1.0.4.jar:2.0.0-sfdc-1.0.4] | |
| at org.jboss.netty.handler.timeout.IdleStateHandler.messageReceived(IdleStateHandler.java:294) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) ~[netty-3.9.1.Final.jar:na] | |
| at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) ~[netty-3.9.1.Final.jar:na] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_76] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_76] | |
| at java.lang.Thread.run(Thread.java:745) [na:1.7.0_76] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-27 18:41:55,966 ERROR [New I/O worker #73] RegionClient: Uncaught error during de-serialization of MultiAction(batch=[PutRequest(table="tsdb", key=[0, 2, 114, 85, 34, 46, -16, 0, 0, 1, 0, 0, -64, 0, 0, 4, 0, 0, 66, 0, 0, 5, 0, 0, -23], family="t", qualifiers=["\xA | |
| 0W"], values=[[0, 0, 0, 0, -101, -56, 35, 75]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=2, region=RegionInfo(table="tsdb", region_name="tsdb,\x00\x02rU\x1B\xA5\x80\x00\x00\x01\x00\x00\xC6\x00\x00\x04\x00\x00?\x00\x00\x05\x00\x00\xEB,1430145994217.d2d866be2678f18d0a193d697a53bea1.", stop_key=[0, 2, 118, 85, 32, 108, -16, 0, 0, 1, 0, 0, -61, 0, 0, 3, 0, 0, 20])), PutRequest(table="tsdb", key=[0, 2, 114, 85, 34, 46, -16, 0, 0, 1, 0, 0, -64, 0, 0, 4, 0, 0, 80, 0, 0, 5, 0, 0, -24], family="t", qualifiers=["\xA0W"], values=[[0, 0, 0, 0, -54, -41, 54, 23]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=1, region=RegionInfo(table="tsdb", region_name="tsdb,\x00\x02rU\x1B\xA5\x80\x00\x00\x01\x00\x00\xC6\x00\x00\x04\x00\x00?\x00\x00\x05\x00\x00\xEB,1430145994217.d2d866be2678f18d0a193d697a53bea1.", stop_key=[0, 2, 118, 85, 32, 108, -16, 0, 0, 1, 0, 0, -61, 0, 0, 3, 0, 0, 20])), ... 1021 RPCs not shown ..., PutRequest(table="tsdb", key=[0, 2, 118, 85, 59, 73, -128, 0, 0, 1, 0, 0, -63, 0, 0, 3, 0, 0, -78], family="t", qualifiers=[[-121, 3]], values=[[0, -110, 0, -84]], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=1, region=RegionInfo(table="tsdb", region_name="tsdb,\x00\x02vU l\xF0\x00\x00\x01\x00\x00\xC3\x00\x00\x03\x00\x00\x14,1430145994217 | |
| .4b08188eabf5b4fe92d3a0b0f8a3fc5b.", stop_key=[0, 2, -81, 85, 31, 97, -64, 0, 0, 1, 0, 0, -28]))]), rpcid=1208 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-04-28 18:07:57,521 WARN [New I/O worker #51] uid.UniqueId - Race condition: tried to assign ID 338 to metrics:yarn.resourcemanager.running_0, but CAS failed on PutRequest(table="tsdb-uid", key="yarn.resourcemanager.running_0", family="id", qualifiers=["metrics"], va | |
| lues=["\x00\x01R"], timestamp=9223372036854775807, lockid=-1, durable=true, bufferable=true, attempt=0, region=null), which indicates this UID must have been allocated concurrently by another TSD or thread. So ID 338 was leaked. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment