Created
October 10, 2025 17:23
-
-
Save paulnicholsen27/18ba5fe34e1162e649627460dd60fb5a to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 25/10/10 13:19:32 ERROR Utils: Aborting task | |
| scala.MatchError: java.lang.NoClassDefFoundError: Could not initialize class org.xerial.snappy.Snappy (of class java.lang.NoClassDefFoundError) | |
| at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2$.attachFilePath(FileDataSourceV2.scala:127) | |
| at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:142) | |
| at org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:695) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown Source) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) | |
| at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) | |
| at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:390) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1323) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:418) | |
| at org.apache.spark.sql.execution.datasources.WriteFilesExec.$anonfun$doExecuteWrite$1(WriteFiles.scala:107) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:901) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:901) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:338) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93) | |
| at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:171) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:147) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:647) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:80) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:77) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:650) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| at java.base/java.lang.Thread.run(Thread.java:840) | |
| 25/10/10 13:19:32 ERROR FileFormatWriter: Job: job_202510101319308908694154349340617_0001, Task: task_202510101319308908694154349340617_0001_m_000010, Task attempt attempt_202510101319308908694154349340617_0001_m_000010_11 aborted. | |
| 25/10/10 13:19:32 ERROR FileFormatWriter: Job: job_202510101319308908694154349340617_0001, Task: task_202510101319308908694154349340617_0001_m_000005, Task attempt attempt_202510101319308908694154349340617_0001_m_000005_6 aborted. | |
| 25/10/10 13:19:32 ERROR FileFormatWriter: Job: job_202510101319308908694154349340617_0001, Task: task_202510101319308908694154349340617_0001_m_000007, Task attempt attempt_202510101319308908694154349340617_0001_m_000007_8 aborted. | |
| 25/10/10 13:19:32 WARN FileOutputCommitter: Could not delete file:/home/nicholsenpm/reference/BTRIS_CB_Color_Coding_09152025_115820/output/btris_order_measurable_clean/_temporary/0/_temporary/attempt_202510101319308908694154349340617_0001_m_000014_15 | |
| 25/10/10 13:19:32 ERROR FileFormatWriter: Job: job_202510101319308908694154349340617_0001, Task: task_202510101319308908694154349340617_0001_m_000014, Task attempt attempt_202510101319308908694154349340617_0001_m_000014_15 aborted. | |
| 25/10/10 13:19:32 WARN TaskSetManager: Lost task 12.0 in stage 1.0 (TID 13) (ncias-p3678-v.nci.nih.gov executor driver): TaskKilled (Stage cancelled: Job aborted due to stage failure: Task 3 in stage 1.0 failed 1 times, most recent failure: Lost task 3.0 in stage 1.0 (TID 4) (ncias-p3678-v.nci.nih.gov executor driver): scala.MatchError: java.lang.OutOfMemoryError: Java heap space (of class java.lang.OutOfMemoryError) | |
| at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2$.attachFilePath(FileDataSourceV2.scala:127) | |
| at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:142) | |
| at org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:695) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown Source) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) | |
| at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) | |
| at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:390) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1323) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:418) | |
| at org.apache.spark.sql.execution.datasources.WriteFilesExec.$anonfun$doExecuteWrite$1(WriteFiles.scala:107) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:901) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:901) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:338) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93) | |
| at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:171) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:147) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:647) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:80) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:77) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:650) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| at java.base/java.lang.Thread.run(Thread.java:840) | |
| Driver stacktrace:) | |
| 25/10/10 13:19:32 WARN TaskSetManager: Lost task 10.0 in stage 1.0 (TID 11) (ncias-p3678-v.nci.nih.gov executor driver): TaskKilled (Stage cancelled: Job aborted due to stage failure: Task 3 in stage 1.0 failed 1 times, most recent failure: Lost task 3.0 in stage 1.0 (TID 4) (ncias-p3678-v.nci.nih.gov executor driver): scala.MatchError: java.lang.OutOfMemoryError: Java heap space (of class java.lang.OutOfMemoryError) | |
| at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2$.attachFilePath(FileDataSourceV2.scala:127) | |
| at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:142) | |
| at org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:695) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown Source) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) | |
| at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) | |
| at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:390) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1323) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:418) | |
| at org.apache.spark.sql.execution.datasources.WriteFilesExec.$anonfun$doExecuteWrite$1(WriteFiles.scala:107) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:901) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:901) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:338) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93) | |
| at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:171) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:147) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:647) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:80) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:77) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:650) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| at java.base/java.lang.Thread.run(Thread.java:840) | |
| Driver stacktrace:) | |
| 25/10/10 13:19:32 WARN TaskSetManager: Lost task 14.0 in stage 1.0 (TID 15) (ncias-p3678-v.nci.nih.gov executor driver): TaskKilled (Stage cancelled: Job aborted due to stage failure: Task 3 in stage 1.0 failed 1 times, most recent failure: Lost task 3.0 in stage 1.0 (TID 4) (ncias-p3678-v.nci.nih.gov executor driver): scala.MatchError: java.lang.OutOfMemoryError: Java heap space (of class java.lang.OutOfMemoryError) | |
| at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2$.attachFilePath(FileDataSourceV2.scala:127) | |
| at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:142) | |
| at org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:695) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown Source) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) | |
| at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) | |
| at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:390) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1323) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:418) | |
| at org.apache.spark.sql.execution.datasources.WriteFilesExec.$anonfun$doExecuteWrite$1(WriteFiles.scala:107) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:901) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:901) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:338) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93) | |
| at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:171) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:147) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:647) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:80) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:77) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:650) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| at java.base/java.lang.Thread.run(Thread.java:840) | |
| Driver stacktrace:) | |
| 25/10/10 13:19:32 WARN TaskSetManager: Lost task 7.0 in stage 1.0 (TID 8) (ncias-p3678-v.nci.nih.gov executor driver): TaskKilled (Stage cancelled: Job aborted due to stage failure: Task 3 in stage 1.0 failed 1 times, most recent failure: Lost task 3.0 in stage 1.0 (TID 4) (ncias-p3678-v.nci.nih.gov executor driver): scala.MatchError: java.lang.OutOfMemoryError: Java heap space (of class java.lang.OutOfMemoryError) | |
| at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2$.attachFilePath(FileDataSourceV2.scala:127) | |
| at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:142) | |
| at org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:695) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown Source) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) | |
| at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) | |
| at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:390) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1323) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:418) | |
| at org.apache.spark.sql.execution.datasources.WriteFilesExec.$anonfun$doExecuteWrite$1(WriteFiles.scala:107) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:901) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:901) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:338) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93) | |
| at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:171) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:147) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:647) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:80) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:77) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:650) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| at java.base/java.lang.Thread.run(Thread.java:840) | |
| Driver stacktrace:) | |
| 25/10/10 13:19:32 WARN TaskSetManager: Lost task 5.0 in stage 1.0 (TID 6) (ncias-p3678-v.nci.nih.gov executor driver): TaskKilled (Stage cancelled: Job aborted due to stage failure: Task 3 in stage 1.0 failed 1 times, most recent failure: Lost task 3.0 in stage 1.0 (TID 4) (ncias-p3678-v.nci.nih.gov executor driver): scala.MatchError: java.lang.OutOfMemoryError: Java heap space (of class java.lang.OutOfMemoryError) | |
| at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2$.attachFilePath(FileDataSourceV2.scala:127) | |
| at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:142) | |
| at org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:695) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown Source) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) | |
| at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) | |
| at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:390) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1323) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:418) | |
| at org.apache.spark.sql.execution.datasources.WriteFilesExec.$anonfun$doExecuteWrite$1(WriteFiles.scala:107) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:901) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:901) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:338) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93) | |
| at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:171) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:147) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:647) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:80) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:77) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:650) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| at java.base/java.lang.Thread.run(Thread.java:840) | |
| Driver stacktrace:) | |
| 25/10/10 13:19:32 WARN TaskSetManager: Lost task 1.0 in stage 1.0 (TID 2) (ncias-p3678-v.nci.nih.gov executor driver): TaskKilled (Stage cancelled: Job aborted due to stage failure: Task 3 in stage 1.0 failed 1 times, most recent failure: Lost task 3.0 in stage 1.0 (TID 4) (ncias-p3678-v.nci.nih.gov executor driver): scala.MatchError: java.lang.OutOfMemoryError: Java heap space (of class java.lang.OutOfMemoryError) | |
| at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2$.attachFilePath(FileDataSourceV2.scala:127) | |
| at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:142) | |
| at org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:695) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown Source) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) | |
| at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) | |
| at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:390) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1323) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:418) | |
| at org.apache.spark.sql.execution.datasources.WriteFilesExec.$anonfun$doExecuteWrite$1(WriteFiles.scala:107) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:901) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:901) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:338) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93) | |
| at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:171) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:147) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:647) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:80) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:77) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:650) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| at java.base/java.lang.Thread.run(Thread.java:840) | |
| Driver stacktrace:) | |
| Traceback (most recent call last): | |
| File "/home/nicholsenpm/reference/BTRIS_CB_Color_Coding_09152025_115820/src/btris_order_measurable_clean/btris_order_measurable_clean.py", line 64, in <module> | |
| main(sys.argv[1:]) | |
| ~~~~^^^^^^^^^^^^^^ | |
| File "/home/nicholsenpm/reference/BTRIS_CB_Color_Coding_09152025_115820/src/btris_order_measurable_clean/btris_order_measurable_clean.py", line 59, in main | |
| result_df.write.mode('overwrite').parquet(output_path) | |
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ | |
| File "/home/nicholsenpm/.local/lib/python3.13/site-packages/pyspark/sql/readwriter.py", line 2003, in parquet | |
| self._jwrite.parquet(path) | |
| ~~~~~~~~~~~~~~~~~~~~^^^^^^ | |
| File "/home/nicholsenpm/.local/lib/python3.13/site-packages/py4j/java_gateway.py", line 1362, in __call__ | |
| return_value = get_return_value( | |
| answer, self.gateway_client, self.target_id, self.name) | |
| File "/home/nicholsenpm/.local/lib/python3.13/site-packages/pyspark/errors/exceptions/captured.py", line 282, in deco | |
| return f(*a, **kw) | |
| File "/home/nicholsenpm/.local/lib/python3.13/site-packages/py4j/protocol.py", line 327, in get_return_value | |
| raise Py4JJavaError( | |
| "An error occurred while calling {0}{1}{2}.\n". | |
| format(target_id, ".", name), value) | |
| py4j.protocol.Py4JJavaError: An error occurred while calling o378.parquet. | |
| : org.apache.spark.SparkException: Job aborted due to stage failure: Task 3 in stage 1.0 failed 1 times, most recent failure: Lost task 3.0 in stage 1.0 (TID 4) (ncias-p3678-v.nci.nih.gov executor driver): scala.MatchError: java.lang.OutOfMemoryError: Java heap space (of class java.lang.OutOfMemoryError) | |
| at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2$.attachFilePath(FileDataSourceV2.scala:127) | |
| at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:142) | |
| at org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:695) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown Source) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) | |
| at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) | |
| at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:390) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1323) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:418) | |
| at org.apache.spark.sql.execution.datasources.WriteFilesExec.$anonfun$doExecuteWrite$1(WriteFiles.scala:107) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:901) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:901) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:338) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93) | |
| at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:171) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:147) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:647) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:80) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:77) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:650) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| at java.base/java.lang.Thread.run(Thread.java:840) | |
| Driver stacktrace: | |
| at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$3(DAGScheduler.scala:2935) | |
| at scala.Option.getOrElse(Option.scala:201) | |
| at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2935) | |
| at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2927) | |
| at scala.collection.immutable.List.foreach(List.scala:334) | |
| at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2927) | |
| at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1295) | |
| at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1295) | |
| at scala.Option.foreach(Option.scala:437) | |
| at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1295) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:3207) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:3141) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:3130) | |
| at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:50) | |
| at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1009) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:2484) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeWrite$4(FileFormatWriter.scala:309) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.writeAndCommit(FileFormatWriter.scala:270) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeWrite(FileFormatWriter.scala:306) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:189) | |
| at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:195) | |
| at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:117) | |
| at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:115) | |
| at org.apache.spark.sql.execution.command.DataWritingCommandExec.executeCollect(commands.scala:129) | |
| at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:155) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:163) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:272) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:125) | |
| at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94) | |
| at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112) | |
| at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:106) | |
| at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:125) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:295) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:124) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:78) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:237) | |
| at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:155) | |
| at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:654) | |
| at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:154) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:169) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:164) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470) | |
| at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:446) | |
| at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:164) | |
| at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:126) | |
| at scala.util.Try$.apply(Try.scala:217) | |
| at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1378) | |
| at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1439) | |
| at org.apache.spark.util.LazyTry.get(LazyTry.scala:58) | |
| at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:131) | |
| at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:192) | |
| at org.apache.spark.sql.classic.DataFrameWriter.runCommand(DataFrameWriter.scala:622) | |
| at org.apache.spark.sql.classic.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:273) | |
| at org.apache.spark.sql.classic.DataFrameWriter.saveInternal(DataFrameWriter.scala:241) | |
| at org.apache.spark.sql.classic.DataFrameWriter.save(DataFrameWriter.scala:118) | |
| at org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:369) | |
| at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) | |
| at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.base/java.lang.reflect.Method.invoke(Method.java:569) | |
| at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) | |
| at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) | |
| at py4j.Gateway.invoke(Gateway.java:282) | |
| at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) | |
| at py4j.commands.CallCommand.execute(CallCommand.java:79) | |
| at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) | |
| at py4j.ClientServerConnection.run(ClientServerConnection.java:108) | |
| at java.base/java.lang.Thread.run(Thread.java:840) | |
| Suppressed: org.apache.spark.util.Utils$OriginalTryStackTraceException: Full stacktrace of original doTryWithCallerStacktrace caller | |
| at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$3(DAGScheduler.scala:2935) | |
| at scala.Option.getOrElse(Option.scala:201) | |
| at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2935) | |
| at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2927) | |
| at scala.collection.immutable.List.foreach(List.scala:334) | |
| at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2927) | |
| at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1295) | |
| at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1295) | |
| at scala.Option.foreach(Option.scala:437) | |
| at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1295) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:3207) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:3141) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:3130) | |
| at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:50) | |
| at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1009) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:2484) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeWrite$4(FileFormatWriter.scala:309) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.writeAndCommit(FileFormatWriter.scala:270) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeWrite(FileFormatWriter.scala:306) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:189) | |
| at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:195) | |
| at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:117) | |
| at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:115) | |
| at org.apache.spark.sql.execution.command.DataWritingCommandExec.executeCollect(commands.scala:129) | |
| at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:155) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:163) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:272) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:125) | |
| at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94) | |
| at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112) | |
| at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:106) | |
| at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:125) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:295) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:124) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:78) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:237) | |
| at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:155) | |
| at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:654) | |
| at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:154) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:169) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:164) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470) | |
| at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:446) | |
| at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:164) | |
| at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:126) | |
| at scala.util.Try$.apply(Try.scala:217) | |
| at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1378) | |
| at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46) | |
| at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46) | |
| ... 20 more | |
| Caused by: scala.MatchError: java.lang.OutOfMemoryError: Java heap space (of class java.lang.OutOfMemoryError) | |
| at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2$.attachFilePath(FileDataSourceV2.scala:127) | |
| at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:142) | |
| at org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:695) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown Source) | |
| at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) | |
| at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) | |
| at org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:390) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1323) | |
| at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:418) | |
| at org.apache.spark.sql.execution.datasources.WriteFilesExec.$anonfun$doExecuteWrite$1(WriteFiles.scala:107) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:901) | |
| at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:901) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:338) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93) | |
| at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:171) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:147) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:647) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:80) | |
| at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:77) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:99) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:650) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) | |
| at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) | |
| ... 1 more |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment