join-on-sparkshell.log

14/11/28 06:34:24 INFO client.DAGClientImpl:    VertexStatus: VertexName: 0 Progress: 100% TotalTasks: 2 Succeeded: 2 Running: 0 Failed: 0 Killed: 0
14/11/28 06:34:24 INFO client.DAGClientImpl:    VertexStatus: VertexName: 1 Progress: 100% TotalTasks: 2 Succeeded: 2 Running: 0 Failed: 0 Killed: 0
14/11/28 06:34:24 INFO client.DAGClientImpl:    VertexStatus: VertexName: 2 Progress: 0% TotalTasks: 6 Succeeded: 0 Running: 4 Failed: 0 Killed: 0
14/11/28 06:34:27 INFO client.DAGClientImpl: DAG: State: FAILED Progress: 40% TotalTasks: 10 Succeeded: 4 Running: 0 Failed: 1 Killed: 5 FailedTaskAttempts: 1
14/11/28 06:34:27 INFO client.DAGClientImpl:    VertexStatus: VertexName: 0 Progress: 100% TotalTasks: 2 Succeeded: 2 Running: 0 Failed: 0 Killed: 0
14/11/28 06:34:27 INFO client.DAGClientImpl:    VertexStatus: VertexName: 1 Progress: 100% TotalTasks: 2 Succeeded: 2 Running: 0 Failed: 0 Killed: 0
14/11/28 06:34:27 INFO client.DAGClientImpl:    VertexStatus: VertexName: 2 Progress: 0% TotalTasks: 6 Succeeded: 0 Running: 0 Failed: 1 Killed: 5 FailedTaskAttempts: 1
14/11/28 06:34:27 INFO client.DAGClientImpl: DAG completed. FinalState=FAILED
14/11/28 06:34:27 INFO tez.DAGBuilder: DAG execution complete
14/11/28 06:34:27 ERROR tez.DAGBuilder: DAG diagnostics: [Vertex failed, vertexName=2, vertexId=vertex_1417036912823_0062_1_02, diagnostics=[Task failed, taskId=task_1417036912823_0062_1_02_000001, diagnostics=[TaskAttempt 0 failed, info=[Error: Failure while running task:java.lang.IllegalStateException: Failed to execute processor for Vertex 2
        at org.apache.spark.tez.SparkTaskProcessor.run(SparkTaskProcessor.scala:60)
        at org.apache.tez.runtime.library.processor.SimpleProcessor.run(SimpleProcessor.java:53)
        at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:324)
        at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:176)
        at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:168)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1614)
        at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.call(TezTaskRunner.java:168)
        at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.call(TezTaskRunner.java:163)
        at java.util.concurrent.FutureTask.run(FutureTask.java:262)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.IllegalStateException: java.lang.NullPointerException
        at org.apache.spark.tez.VertexResultTask.runTask(VertexResultTask.scala:95)
        at org.apache.spark.tez.SparkUtils$.runTask(SparkUtils.scala:149)
        at org.apache.spark.tez.SparkTaskProcessor.doRun(SparkTaskProcessor.scala:97)
        at org.apache.spark.tez.SparkTaskProcessor.run(SparkTaskProcessor.scala:56)
        ... 13 more
Caused by: java.lang.NullPointerException
        at org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:156)
        at org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:63)
        at org.apache.spark.util.collection.Spillable$class.maybeSpill(Spillable.scala:77)
        at org.apache.spark.util.collection.ExternalAppendOnlyMap.maybeSpill(ExternalAppendOnlyMap.scala:63)
        at org.apache.spark.util.collection.ExternalAppendOnlyMap.insertAll(ExternalAppendOnlyMap.scala:131)
        at org.apache.spark.rdd.CoGroupedRDD$$anonfun$compute$5.apply(CoGroupedRDD.scala:160)
        at org.apache.spark.rdd.CoGroupedRDD$$anonfun$compute$5.apply(CoGroupedRDD.scala:159)
        at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:772)
        at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
        at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
        at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:771)
        at org.apache.spark.rdd.CoGroupedRDD.compute(CoGroupedRDD.scala:159)
        at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:260)
        at org.apache.spark.rdd.RDD.iterator(RDD.scala:227)
        at org.apache.spark.rdd.MappedValuesRDD.compute(MappedValuesRDD.scala:31)
        at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:260)
        at org.apache.spark.rdd.RDD.iterator(RDD.scala:227)
        at org.apache.spark.rdd.FlatMappedValuesRDD.compute(FlatMappedValuesRDD.scala:31)
        at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:260)
        at org.apache.spark.rdd.RDD.iterator(RDD.scala:227)
        at org.apache.spark.rdd.MappedRDD.compute(MappedRDD.scala:31)
        at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:260)
        at org.apache.spark.rdd.RDD.iterator(RDD.scala:227)
        at org.apache.spark.tez.VertexResultTask.runTask(VertexResultTask.scala:87)
        ... 16 more
]], Vertex failed as one or more tasks failed. failedTasks:1, Vertex vertex_1417036912823_0062_1_02 [2] killed/failed due to:null], DAG failed due to vertex failure. failedVertices:1 killedVertices:0]