Created
February 14, 2018 19:14
-
-
Save ottomata/2a46db62f415b651947569ff42397296 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Driver stacktrace: | |
| 18/02/14 19:12:04 WARN ExecutorAllocationManager: No stages are running, but numRunningTasks != 0 | |
| 18/02/14 19:12:04 INFO DAGScheduler: Job 3 failed: insertInto at SparkJsonToHive.scala:186, took 0.537105 s | |
| 18/02/14 19:12:04 WARN TaskSetManager: Lost task 0.0 in stage 3.0 (TID 46, localhost, executor driver): TaskKilled (killed intentionally) | |
| 18/02/14 19:12:04 ERROR JsonRefine: Failed refinement of JSON dataset hdfs://analytics-hadoop/user/otto/el2/eventlogging_MobileWikiAppSessions/hourly/2018/02/01/01 -> MobileWikiAppSessions (year=2018,month=2,day=1,hour=1). | |
| org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 3.0 failed 1 times, most recent failure: Lost task 1.0 in stage 3.0 (TID 47, localhost, executor driver): scala.MatchError: null | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StringConverter$.toCatalystImpl(CatalystTypeConverters.scala:295) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StringConverter$.toCatalystImpl(CatalystTypeConverters.scala:294) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$CatalystTypeConverter.toCatalyst(CatalystTypeConverters.scala:97) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StructConverter.toCatalystImpl(CatalystTypeConverters.scala:260) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StructConverter.toCatalystImpl(CatalystTypeConverters.scala:250) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$CatalystTypeConverter.toCatalyst(CatalystTypeConverters.scala:102) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StructConverter.toCatalystImpl(CatalystTypeConverters.scala:260) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StructConverter.toCatalystImpl(CatalystTypeConverters.scala:250) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$CatalystTypeConverter.toCatalyst(CatalystTypeConverters.scala:102) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$$anonfun$createToCatalystConverter$2.apply(CatalystTypeConverters.scala:401) | |
| at org.apache.spark.sql.SQLContext$$anonfun$6.apply(SQLContext.scala:492) | |
| at org.apache.spark.sql.SQLContext$$anonfun$6.apply(SQLContext.scala:492) | |
| at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) | |
| at scala.collection.Iterator$class.foreach(Iterator.scala:727) | |
| at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) | |
| at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.org$apache$spark$sql$hive$execution$InsertIntoHiveTable$$writeToFile$1(InsertIntoHiveTable.scala:104) | |
| at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$saveAsHiveFile$3.apply(InsertIntoHiveTable.scala:84) | |
| at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$saveAsHiveFile$3.apply(InsertIntoHiveTable.scala:84) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:89) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:242) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Driver stacktrace: | |
| at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1433) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1421) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1420) | |
| at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) | |
| at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) | |
| at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1420) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799) | |
| at scala.Option.foreach(Option.scala:236) | |
| at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:799) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1644) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1603) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1592) | |
| at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) | |
| at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:620) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:1840) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:1853) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:1930) | |
| at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.saveAsHiveFile(InsertIntoHiveTable.scala:84) | |
| at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.sideEffectResult$lzycompute(InsertIntoHiveTable.scala:201) | |
| at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.sideEffectResult(InsertIntoHiveTable.scala:127) | |
| at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.doExecute(InsertIntoHiveTable.scala:276) | |
| at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:132) | |
| at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:130) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:130) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:55) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:55) | |
| at org.apache.spark.sql.DataFrameWriter.insertInto(DataFrameWriter.scala:189) | |
| at org.apache.spark.sql.DataFrameWriter.insertInto(DataFrameWriter.scala:166) | |
| at org.wikimedia.analytics.refinery.job.jsonrefine.SparkJsonToHive$.apply(SparkJsonToHive.scala:186) | |
| at org.wikimedia.analytics.refinery.job.jsonrefine.JsonRefine$$anonfun$refineTargets$1.apply(JsonRefine.scala:568) | |
| at org.wikimedia.analytics.refinery.job.jsonrefine.JsonRefine$$anonfun$refineTargets$1.apply(JsonRefine.scala:558) | |
| at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) | |
| at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) | |
| at scala.collection.mutable.ArraySeq.foreach(ArraySeq.scala:73) | |
| at scala.collection.TraversableLike$class.map(TraversableLike.scala:244) | |
| at scala.collection.AbstractTraversable.map(Traversable.scala:105) | |
| at org.wikimedia.analytics.refinery.job.jsonrefine.JsonRefine$.refineTargets(JsonRefine.scala:558) | |
| at org.wikimedia.analytics.refinery.job.jsonrefine.JsonRefine$$anonfun$34.apply(JsonRefine.scala:426) | |
| at org.wikimedia.analytics.refinery.job.jsonrefine.JsonRefine$$anonfun$34.apply(JsonRefine.scala:420) | |
| at scala.collection.parallel.AugmentedIterableIterator$class.map2combiner(RemainsIterator.scala:120) | |
| at scala.collection.parallel.immutable.ParHashMap$ParHashMapIterator.map2combiner(ParHashMap.scala:79) | |
| at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1057) | |
| at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply$mcV$sp(Tasks.scala:54) | |
| at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply(Tasks.scala:53) | |
| at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply(Tasks.scala:53) | |
| at scala.collection.parallel.Task$class.tryLeaf(Tasks.scala:56) | |
| at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1054) | |
| at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask$class.compute(Tasks.scala:165) | |
| at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:514) | |
| at scala.concurrent.forkjoin.RecursiveAction.exec(RecursiveAction.java:160) | |
| at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) | |
| at scala.concurrent.forkjoin.ForkJoinTask.doJoin(ForkJoinTask.java:341) | |
| at scala.concurrent.forkjoin.ForkJoinTask.join(ForkJoinTask.java:673) | |
| at scala.collection.parallel.ForkJoinTasks$WrappedTask$class.sync(Tasks.scala:444) | |
| at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:514) | |
| at scala.collection.parallel.ForkJoinTasks$class.executeAndWaitResult(Tasks.scala:492) | |
| at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:64) | |
| at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:961) | |
| at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply$mcV$sp(Tasks.scala:54) | |
| at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply(Tasks.scala:53) | |
| at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply(Tasks.scala:53) | |
| at scala.collection.parallel.Task$class.tryLeaf(Tasks.scala:56) | |
| at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:956) | |
| at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask$class.compute(Tasks.scala:165) | |
| at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:514) | |
| at scala.concurrent.forkjoin.RecursiveAction.exec(RecursiveAction.java:160) | |
| at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) | |
| at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) | |
| at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) | |
| at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) | |
| Caused by: scala.MatchError: null | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StringConverter$.toCatalystImpl(CatalystTypeConverters.scala:295) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StringConverter$.toCatalystImpl(CatalystTypeConverters.scala:294) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$CatalystTypeConverter.toCatalyst(CatalystTypeConverters.scala:97) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StructConverter.toCatalystImpl(CatalystTypeConverters.scala:260) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StructConverter.toCatalystImpl(CatalystTypeConverters.scala:250) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$CatalystTypeConverter.toCatalyst(CatalystTypeConverters.scala:102) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StructConverter.toCatalystImpl(CatalystTypeConverters.scala:260) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$StructConverter.toCatalystImpl(CatalystTypeConverters.scala:250) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$CatalystTypeConverter.toCatalyst(CatalystTypeConverters.scala:102) | |
| at org.apache.spark.sql.catalyst.CatalystTypeConverters$$anonfun$createToCatalystConverter$2.apply(CatalystTypeConverters.scala:401) | |
| at org.apache.spark.sql.SQLContext$$anonfun$6.apply(SQLContext.scala:492) | |
| at org.apache.spark.sql.SQLContext$$anonfun$6.apply(SQLContext.scala:492) | |
| at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) | |
| at scala.collection.Iterator$class.foreach(Iterator.scala:727) | |
| at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) | |
| at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.org$apache$spark$sql$hive$execution$InsertIntoHiveTable$$writeToFile$1(InsertIntoHiveTable.scala:104) | |
| at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$saveAsHiveFile$3.apply(InsertIntoHiveTable.scala:84) | |
| at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$saveAsHiveFile$3.apply(InsertIntoHiveTable.scala:84) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:89) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:242) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment