Created
January 4, 2018 18:02
-
-
Save ianmilligan1/8d8103c4207fbcf7045d609ee05a9ce2 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2018-01-04 18:00:34,213 [Executor task launch worker for task 119] ERROR Executor - Exception in task 119.0 in stage 0.0 (TID 119) | |
| java.lang.OutOfMemoryError: Java heap space | |
| at java.util.Arrays.copyOf(Arrays.java:3332) | |
| at java.lang.StringCoding.safeTrim(StringCoding.java:89) | |
| at java.lang.StringCoding.access$100(StringCoding.java:50) | |
| at java.lang.StringCoding$StringDecoder.decode(StringCoding.java:154) | |
| at java.lang.StringCoding.decode(StringCoding.java:193) | |
| at java.lang.StringCoding.decode(StringCoding.java:254) | |
| at java.lang.String.<init>(String.java:546) | |
| at java.lang.String.<init>(String.java:566) | |
| at io.archivesunleashed.data.WarcRecordUtils.getWarcResponseMimeType(WarcRecordUtils.java:102) | |
| at io.archivesunleashed.spark.archive.io.ArchiveRecord.<init>(ArchiveRecord.scala:74) | |
| at io.archivesunleashed.spark.matchbox.RecordLoader$$anonfun$2.apply(RecordLoader.scala:37) | |
| at io.archivesunleashed.spark.matchbox.RecordLoader$$anonfun$2.apply(RecordLoader.scala:37) | |
| at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) | |
| at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:462) | |
| at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:461) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) | |
| at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:191) | |
| at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63) | |
| at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96) | |
| at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| 2018-01-04 18:00:34,246 [Executor task launch worker for task 119] ERROR SparkUncaughtExceptionHandler - Uncaught exception in thread Thread[Executor task launch worker for task 119,5,main] | |
| java.lang.OutOfMemoryError: Java heap space | |
| at java.util.Arrays.copyOf(Arrays.java:3332) | |
| at java.lang.StringCoding.safeTrim(StringCoding.java:89) | |
| at java.lang.StringCoding.access$100(StringCoding.java:50) | |
| at java.lang.StringCoding$StringDecoder.decode(StringCoding.java:154) | |
| at java.lang.StringCoding.decode(StringCoding.java:193) | |
| at java.lang.StringCoding.decode(StringCoding.java:254) | |
| at java.lang.String.<init>(String.java:546) | |
| at java.lang.String.<init>(String.java:566) | |
| at io.archivesunleashed.data.WarcRecordUtils.getWarcResponseMimeType(WarcRecordUtils.java:102) | |
| at io.archivesunleashed.spark.archive.io.ArchiveRecord.<init>(ArchiveRecord.scala:74) | |
| at io.archivesunleashed.spark.matchbox.RecordLoader$$anonfun$2.apply(RecordLoader.scala:37) | |
| at io.archivesunleashed.spark.matchbox.RecordLoader$$anonfun$2.apply(RecordLoader.scala:37) | |
| at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) | |
| at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:462) | |
| at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:461) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) | |
| at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:191) | |
| at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63) | |
| at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96) | |
| at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| 2018-01-04 18:00:34,314 [task-result-getter-0] WARN TaskSetManager - Lost task 119.0 in stage 0.0 (TID 119, localhost, executor driver): java.lang.OutOfMemoryError: Java heap space | |
| at java.util.Arrays.copyOf(Arrays.java:3332) | |
| at java.lang.StringCoding.safeTrim(StringCoding.java:89) | |
| at java.lang.StringCoding.access$100(StringCoding.java:50) | |
| at java.lang.StringCoding$StringDecoder.decode(StringCoding.java:154) | |
| at java.lang.StringCoding.decode(StringCoding.java:193) | |
| at java.lang.StringCoding.decode(StringCoding.java:254) | |
| at java.lang.String.<init>(String.java:546) | |
| at java.lang.String.<init>(String.java:566) | |
| at io.archivesunleashed.data.WarcRecordUtils.getWarcResponseMimeType(WarcRecordUtils.java:102) | |
| at io.archivesunleashed.spark.archive.io.ArchiveRecord.<init>(ArchiveRecord.scala:74) | |
| at io.archivesunleashed.spark.matchbox.RecordLoader$$anonfun$2.apply(RecordLoader.scala:37) | |
| at io.archivesunleashed.spark.matchbox.RecordLoader$$anonfun$2.apply(RecordLoader.scala:37) | |
| at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) | |
| at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:462) | |
| at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:461) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) | |
| at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:191) | |
| at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63) | |
| at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96) | |
| at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| 2018-01-04 18:00:34,320 [task-result-getter-0] ERROR TaskSetManager - Task 119 in stage 0.0 failed 1 times; aborting job | |
| 2018-01-04 18:00:34,323 [dag-scheduler-event-loop] ERROR LiveListenerBus - SparkListenerBus has already stopped! Dropping event SparkListenerTaskEnd(0,0,ShuffleMapTask,ExceptionFailure(java.lang.OutOfMemoryError,Java heap space,[Ljava.lang.StackTraceElement;@55e60c01,java.lang.OutOfMemoryError: Java heap space | |
| at java.util.Arrays.copyOf(Arrays.java:3332) | |
| at java.lang.StringCoding.safeTrim(StringCoding.java:89) | |
| at java.lang.StringCoding.access$100(StringCoding.java:50) | |
| at java.lang.StringCoding$StringDecoder.decode(StringCoding.java:154) | |
| at java.lang.StringCoding.decode(StringCoding.java:193) | |
| at java.lang.StringCoding.decode(StringCoding.java:254) | |
| at java.lang.String.<init>(String.java:546) | |
| at java.lang.String.<init>(String.java:566) | |
| at io.archivesunleashed.data.WarcRecordUtils.getWarcResponseMimeType(WarcRecordUtils.java:102) | |
| at io.archivesunleashed.spark.archive.io.ArchiveRecord.<init>(ArchiveRecord.scala:74) | |
| at io.archivesunleashed.spark.matchbox.RecordLoader$$anonfun$2.apply(RecordLoader.scala:37) | |
| at io.archivesunleashed.spark.matchbox.RecordLoader$$anonfun$2.apply(RecordLoader.scala:37) | |
| at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) | |
| at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:462) | |
| at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:461) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) | |
| at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:191) | |
| at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63) | |
| at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96) | |
| at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| ,Some(org.apache.spark.ThrowableSerializationWrapper@6d89cd69),Vector(AccumulableInfo(2,Some(internal.metrics.executorRunTime),Some(642516),None,true,true,None), AccumulableInfo(4,Some(internal.metrics.resultSize),Some(0),None,true,true,None), AccumulableInfo(5,Some(internal.metrics.jvmGCTime),Some(112293),None,true,true,None), AccumulableInfo(20,Some(internal.metrics.input.bytesRead),Some(1096652444),None,true,true,None), AccumulableInfo(21,Some(internal.metrics.input.recordsRead),Some(317),None,true,true,None)),Vector(LongAccumulator(id: 2, name: Some(internal.metrics.executorRunTime), value: 642516), LongAccumulator(id: 4, name: Some(internal.metrics.resultSize), value: 0), LongAccumulator(id: 5, name: Some(internal.metrics.jvmGCTime), value: 112293), LongAccumulator(id: 20, name: Some(internal.metrics.input.bytesRead), value: 1096652444), LongAccumulator(id: 21, name: Some(internal.metrics.input.recordsRead), value: 317))),org.apache.spark.scheduler.TaskInfo@20a54bcb,org.apache.spark.executor.TaskMetrics@3441128b) | |
| 2018-01-04 18:00:34,334 [dag-scheduler-event-loop] ERROR LiveListenerBus - SparkListenerBus has already stopped! Dropping event SparkListenerStageCompleted(org.apache.spark.scheduler.StageInfo@9f50a5) | |
| 2018-01-04 18:00:34,340 [dag-scheduler-event-loop] ERROR LiveListenerBus - SparkListenerBus has already stopped! Dropping event SparkListenerJobEnd(0,1515088834338,JobFailed(org.apache.spark.SparkException: Job aborted due to stage failure: Task 119 in stage 0.0 failed 1 times, most recent failure: Lost task 119.0 in stage 0.0 (TID 119, localhost, executor driver): java.lang.OutOfMemoryError: Java heap space | |
| at java.util.Arrays.copyOf(Arrays.java:3332) | |
| at java.lang.StringCoding.safeTrim(StringCoding.java:89) | |
| at java.lang.StringCoding.access$100(StringCoding.java:50) | |
| at java.lang.StringCoding$StringDecoder.decode(StringCoding.java:154) | |
| at java.lang.StringCoding.decode(StringCoding.java:193) | |
| at java.lang.StringCoding.decode(StringCoding.java:254) | |
| at java.lang.String.<init>(String.java:546) | |
| at java.lang.String.<init>(String.java:566) | |
| at io.archivesunleashed.data.WarcRecordUtils.getWarcResponseMimeType(WarcRecordUtils.java:102) | |
| at io.archivesunleashed.spark.archive.io.ArchiveRecord.<init>(ArchiveRecord.scala:74) | |
| at io.archivesunleashed.spark.matchbox.RecordLoader$$anonfun$2.apply(RecordLoader.scala:37) | |
| at io.archivesunleashed.spark.matchbox.RecordLoader$$anonfun$2.apply(RecordLoader.scala:37) | |
| at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) | |
| at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:462) | |
| at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:461) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) | |
| at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:191) | |
| at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63) | |
| at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96) | |
| at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Driver stacktrace:)) | |
| 2018-01-04 18:00:34,413 [Thread-1] ERROR ShutdownHookManager - Exception while deleting Spark temp dir: /tmp/spark-00c001cc-545a-4c8b-8e77-a06133ce92d4 | |
| java.io.IOException: Failed to delete: /tmp/spark-00c001cc-545a-4c8b-8e77-a06133ce92d4 | |
| at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:1010) | |
| at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:65) | |
| at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:62) | |
| at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33) | |
| at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186) | |
| at org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:62) | |
| at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:216) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$appl | |
| y$mcV$sp$1.apply(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1951) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:188) | |
| at scala.util.Try$.apply(Try.scala:192) | |
| at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:178) | |
| at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) | |
| ubuntu@AUT-Troubleshooter:~/aut$ | |
| at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Driver stacktrace:)) | |
| 2018-01-04 18:00:34,413 [Thread-1] ERROR ShutdownHookManager - Exception while deleting Spark temp dir: /tmp/spark-00c001cc-545a-4c8b-8e77-a06133ce92d4 | |
| java.io.IOException: Failed to delete: /tmp/spark-00c001cc-545a-4c8b-8e77-a06133ce92d4 | |
| at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:1010) | |
| at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:65) | |
| at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:62) | |
| at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33) | |
| at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186) | |
| at org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:62) | |
| at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:216) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1951) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:188) | |
| at scala.util.Try$.apply(Try.scala:192) | |
| at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:188) | |
| at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:178) | |
| at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment