yarn_race.txt

16/02/08 11:28:19 INFO ApplicationMasterService: Starting Application Master
16/02/08 11:28:19 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
16/02/08 11:28:19 INFO RMProxy: Connecting to ResourceManager at dm01/10.14.1.38:8030
16/02/08 11:28:20 INFO ApplicationMasterService: Registered ApplicationMaster: maximumCapability { memory: 15360 virtual_cores: 32 } queue: "default" scheduler_resource_types: MEMORY
16/02/08 11:28:20 INFO NMClientAsyncImpl: Upper bound of the thread pool size is 500
16/02/08 11:28:20 INFO ContainerManagementProtocolProxy: yarn.client.max-cached-nodemanagers-proxies : 0
16/02/08 11:28:22 INFO AMRMClientImpl: Received new token for : ip-10-14-130-102.ec2.internal:60563
16/02/08 11:28:22 INFO AMRMClientImpl: Received new token for : ip-10-14-129-101.ec2.internal:48201
16/02/08 11:28:22 INFO AMRMClientImpl: Received new token for : ip-10-14-130-107.ec2.internal:54722
16/02/08 11:28:22 INFO ApplicationMasterService: Allocating 3 container(s)
16/02/08 11:28:22 INFO ContainerTracker: Launching a container 'container_1447129958864_0174_01_000002' on node: ip-10-14-130-102.ec2.internal:60563
16/02/08 11:28:22 INFO ContainerTracker: Launching a container 'container_1447129958864_0174_01_000003' on node: ip-10-14-129-101.ec2.internal:48201
16/02/08 11:28:22 INFO ContainerTracker: Launching a container 'container_1447129958864_0174_01_000004' on node: ip-10-14-130-107.ec2.internal:54722
16/02/08 11:28:22 INFO NMClientAsyncImpl: Processing Event EventType: START_CONTAINER for Container container_1447129958864_0174_01_000002
16/02/08 11:28:22 INFO NMClientAsyncImpl: Processing Event EventType: START_CONTAINER for Container container_1447129958864_0174_01_000003
16/02/08 11:28:22 INFO NMClientAsyncImpl: Processing Event EventType: START_CONTAINER for Container container_1447129958864_0174_01_000004
16/02/08 11:28:22 INFO ContainerManagementProtocolProxy: Opening proxy : ip-10-14-129-101.ec2.internal:48201
16/02/08 11:28:22 INFO ContainerManagementProtocolProxy: Opening proxy : ip-10-14-130-102.ec2.internal:60563
16/02/08 11:28:22 INFO ContainerManagementProtocolProxy: Opening proxy : ip-10-14-130-107.ec2.internal:54722
16/02/08 11:28:22 WARN ContainerTracker: Got an error for starting a container: container_1447129958864_0174_01_000002
org.apache.hadoop.yarn.exceptions.YarnException: java.lang.IllegalArgumentException
	at org.apache.hadoop.yarn.ipc.RPCUtil.getRemoteException(RPCUtil.java:38)
	at org.apache.hadoop.yarn.client.api.impl.NMClientImpl.startContainer(NMClientImpl.java:224)
	at org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl$StatefulContainer$StartContainerTransition.transition(NMClientAsyncImpl.java:378)
	at org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl$StatefulContainer$StartContainerTransition.transition(NMClientAsyncImpl.java:363)
	at org.apache.hadoop.yarn.state.StateMachineFactory$MultipleInternalArc.doTransition(StateMachineFactory.java:385)
	at org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:302)
	at org.apache.hadoop.yarn.state.StateMachineFactory.access$300(StateMachineFactory.java:46)
	at org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:448)
	at org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl$StatefulContainer.handle(NMClientAsyncImpl.java:498)
	at org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl$ContainerEventProcessor.run(NMClientAsyncImpl.java:557)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.IllegalArgumentException
	at java.nio.Buffer.position(Buffer.java:244)
	at java.nio.HeapByteBuffer.get(HeapByteBuffer.java:153)
	at java.nio.ByteBuffer.get(ByteBuffer.java:715)
	at com.google.protobuf.ByteString.copyFrom(ByteString.java:217)
	at com.google.protobuf.ByteString.copyFrom(ByteString.java:229)
	at org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils.convertToProtoFormat(ProtoUtils.java:198)
	at org.apache.hadoop.yarn.api.records.impl.pb.ContainerLaunchContextPBImpl.convertToProtoFormat(ContainerLaunchContextPBImpl.java:101)
	at org.apache.hadoop.yarn.api.records.impl.pb.ContainerLaunchContextPBImpl.mergeLocalToBuilder(ContainerLaunchContextPBImpl.java:109)
	at org.apache.hadoop.yarn.api.records.impl.pb.ContainerLaunchContextPBImpl.mergeLocalToProto(ContainerLaunchContextPBImpl.java:128)
	at org.apache.hadoop.yarn.api.records.impl.pb.ContainerLaunchContextPBImpl.getProto(ContainerLaunchContextPBImpl.java:70)
	at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestPBImpl.convertToProtoFormat(StartContainerRequestPBImpl.java:156)
	at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestPBImpl.mergeLocalToBuilder(StartContainerRequestPBImpl.java:85)
	at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestPBImpl.mergeLocalToProto(StartContainerRequestPBImpl.java:95)
	at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestPBImpl.getProto(StartContainerRequestPBImpl.java:57)
	at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl.convertToProtoFormat(StartContainersRequestPBImpl.java:137)
	at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl.addLocalRequestsToProto(StartContainersRequestPBImpl.java:97)
	at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl.mergeLocalToBuilder(StartContainersRequestPBImpl.java:79)
	at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl.mergeLocalToProto(StartContainersRequestPBImpl.java:72)
	at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl.getProto(StartContainersRequestPBImpl.java:48)
	at org.apache.hadoop.yarn.api.impl.pb.client.ContainerManagementProtocolPBClientImpl.startContainers(ContainerManagementProtocolPBClientImpl.java:94)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:497)
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
	at com.sun.proxy.$Proxy10.startContainers(Unknown Source)
	at org.apache.hadoop.yarn.client.api.impl.NMClientImpl.startContainer(NMClientImpl.java:203)
	... 11 more
16/02/08 11:28:22 INFO ContainerTracker: Starting a container: container_1447129958864_0174_01_000004
16/02/08 11:28:22 INFO ContainerTracker: Starting a container: container_1447129958864_0174_01_000003
16/02/08 11:28:22 INFO NMClientAsyncImpl: Processing Event EventType: QUERY_CONTAINER for Container container_1447129958864_0174_01_000004
16/02/08 11:28:22 INFO NMClientAsyncImpl: Processing Event EventType: QUERY_CONTAINER for Container container_1447129958864_0174_01_000003
16/02/08 11:28:22 INFO ContainerManagementProtocolProxy: Opening proxy : ip-10-14-130-107.ec2.internal:54722
16/02/08 11:28:22 INFO ContainerManagementProtocolProxy: Opening proxy : ip-10-14-129-101.ec2.internal:48201
16/02/08 11:28:23 INFO ApplicationMasterService: 2 container(s) have completed
16/02/08 11:28:23 INFO ApplicationMasterService: Container container_1447129958864_0174_01_000003 finished succesfully
16/02/08 11:28:23 INFO ApplicationMasterService: Container container_1447129958864_0174_01_000004 finished succesfully