Skip to content

Instantly share code, notes, and snippets.

@shyamsalimkumar
Created August 17, 2015 12:46
Show Gist options
  • Save shyamsalimkumar/f874fb7c557ae47f8008 to your computer and use it in GitHub Desktop.
Save shyamsalimkumar/f874fb7c557ae47f8008 to your computer and use it in GitHub Desktop.
Spark serialization error
15/08/17 16:24:21 ERROR Executor: Exception in task 0.0 in stage 7.0 (TID 7)
java.io.NotSerializableException: com.acme.avro.Swipe
Serialization stack:
- object not serializable (class: com.acme.avro.Swipe, value: {"center_id": 1, "gate_id": 2, "employee_id": 1597, "card_id": 12575632, "employee_name": "Anju Chandran", "swipe_time": 0, "coming_in": true})
- field (class: scala.Tuple2, name: _2, type: class java.lang.Object)
- object (class scala.Tuple2, ([B@6c0e6da7,{"center_id": 1, "gate_id": 2, "employee_id": 1597, "card_id": 12575632, "employee_name": "Anju Chandran", "swipe_time": 0, "coming_in": true}))
- element of array (index: 0)
- array (class [Lscala.Tuple2;, size 1)
at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40)
at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:47)
at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:81)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:236)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
15/08/17 16:24:21 ERROR TaskSetManager: Task 0.0 in stage 7.0 (TID 7) had a not serializable result: com.acme.avro.Swipe
Serialization stack:
- object not serializable (class: com.acme.avro.Swipe, value: {"center_id": 1, "gate_id": 2, "employee_id": 1597, "card_id": 12575632, "employee_name": "Anju Chandran", "swipe_time": 0, "coming_in": true})
- field (class: scala.Tuple2, name: _2, type: class java.lang.Object)
- object (class scala.Tuple2, ([B@6c0e6da7,{"center_id": 1, "gate_id": 2, "employee_id": 1597, "card_id": 12575632, "employee_name": "Anju Chandran", "swipe_time": 0, "coming_in": true}))
- element of array (index: 0)
- array (class [Lscala.Tuple2;, size 1); not retrying
15/08/17 16:26:24 ERROR JobScheduler: Error running job streaming job 1439808861000 ms.0
org.apache.spark.SparkException: Job aborted due to stage failure: Task 0.0 in stage 7.0 (TID 7) had a not serializable result: com.acme.avro.Swipe
Serialization stack:
- object not serializable (class: com.acme.avro.Swipe, value: {"center_id": 1, "gate_id": 2, "employee_id": 1597, "card_id": 12575632, "employee_name": "Anju Chandran", "swipe_time": 0, "coming_in": true})
- field (class: scala.Tuple2, name: _2, type: class java.lang.Object)
- object (class scala.Tuple2, ([B@6c0e6da7,{"center_id": 1, "gate_id": 2, "employee_id": 1597, "card_id": 12575632, "employee_name": "Anju Chandran", "swipe_time": 0, "coming_in": true}))
- element of array (index: 0)
- array (class [Lscala.Tuple2;, size 1)
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1273)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1264)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1263)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1263)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:730)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:730)
at scala.Option.foreach(Option.scala:236)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:730)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1457)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1418)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 0.0 in stage 7.0 (TID 7) had a not serializable result: com.acme.avro.Swipe
Serialization stack:
- object not serializable (class: com.acme.avro.Swipe, value: {"center_id": 1, "gate_id": 2, "employee_id": 1597, "card_id": 12575632, "employee_name": "Anju Chandran", "swipe_time": 0, "coming_in": true})
- field (class: scala.Tuple2, name: _2, type: class java.lang.Object)
- object (class scala.Tuple2, ([B@6c0e6da7,{"center_id": 1, "gate_id": 2, "employee_id": 1597, "card_id": 12575632, "employee_name": "Anju Chandran", "swipe_time": 0, "coming_in": true}))
- element of array (index: 0)
- array (class [Lscala.Tuple2;, size 1)
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1273)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1264)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1263)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1263)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:730)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:730)
at scala.Option.foreach(Option.scala:236)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:730)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1457)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1418)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment