Created
August 6, 2021 02:46
-
-
Save nsivabalan/e4ad02757f2471b821e5c69d23889ac3 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| scala> spark.time(df1.write.format("hudi").option("hoodie.bulkinsert.shuffle.parallelism","120").option(PRECOMBINE_FIELD.key(), "created_at").option(RECORDKEY_FIELD.key(), "id").option(PARTITIONPATH_FIELD.key(), "date_col").option("hoodie.parquet.compression.codec", "SNAPPY").option(OPERATION.key(),"bulk_insert"). | |
| | option("hoodie.datasource.write.row.writer.enable", "true").option(TABLE_NAME.key(), "hudi_3").mode(Overwrite).save("s3a://siva-test-bucket-june-16/hudi_testing/hudi_3/")) | |
| 21/08/06 02:43:37 WARN HoodieSparkSqlWriter$: hoodie table at s3a://siva-test-bucket-june-16/hudi_testing/hudi_3 already exists. Deleting existing data & overwriting with new data. | |
| 21/08/06 02:44:20 ERROR AppendDataExec: Data source write support org.apache.hudi.spark3.internal.HoodieDataSourceInternalBatchWrite@2237dc21 is aborting. | |
| 21/08/06 02:44:20 ERROR DataSourceInternalWriterHelper: Commit 20210806024337 aborted | |
| 21/08/06 02:44:25 ERROR AppendDataExec: Data source write support org.apache.hudi.spark3.internal.HoodieDataSourceInternalBatchWrite@2237dc21 failed to abort. | |
| org.apache.spark.SparkException: Writing job failed. | |
| at org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2(WriteToDataSourceV2Exec.scala:408) | |
| at org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2$(WriteToDataSourceV2Exec.scala:361) | |
| at org.apache.spark.sql.execution.datasources.v2.AppendDataExec.writeWithV2(WriteToDataSourceV2Exec.scala:253) | |
| at org.apache.spark.sql.execution.datasources.v2.AppendDataExec.run(WriteToDataSourceV2Exec.scala:259) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result$lzycompute(V2CommandExec.scala:39) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result(V2CommandExec.scala:39) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.doExecute(V2CommandExec.scala:54) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121) | |
| at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:963) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) | |
| at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:963) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:354) | |
| at org.apache.hudi.HoodieSparkSqlWriter$.bulkInsertAsRow(HoodieSparkSqlWriter.scala:378) | |
| at org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:149) | |
| at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:165) | |
| at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:46) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:90) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121) | |
| at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:963) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) | |
| at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:963) | |
| at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:415) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:399) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:288) | |
| at $anonfun$res3$1(<console>:42) | |
| at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) | |
| at org.apache.spark.sql.SparkSession.time(SparkSession.scala:668) | |
| ... 58 elided | |
| Caused by: org.apache.hudi.exception.HoodieException: Task not serializable | |
| at org.apache.hudi.internal.DataSourceInternalWriterHelper.commit(DataSourceInternalWriterHelper.java:86) | |
| at org.apache.hudi.spark3.internal.HoodieDataSourceInternalBatchWrite.commit(HoodieDataSourceInternalBatchWrite.java:93) | |
| at org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2(WriteToDataSourceV2Exec.scala:396) | |
| ... 106 more | |
| Caused by: org.apache.spark.SparkException: Task not serializable | |
| at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:416) | |
| at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:406) | |
| at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:162) | |
| at org.apache.spark.SparkContext.clean(SparkContext.scala:2362) | |
| at org.apache.spark.rdd.RDD.$anonfun$map$1(RDD.scala:396) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) | |
| at org.apache.spark.rdd.RDD.withScope(RDD.scala:388) | |
| at org.apache.spark.rdd.RDD.map(RDD.scala:395) | |
| at org.apache.spark.api.java.JavaRDDLike.mapToPair(JavaRDDLike.scala:117) | |
| at org.apache.spark.api.java.JavaRDDLike.mapToPair$(JavaRDDLike.scala:115) | |
| at org.apache.spark.api.java.AbstractJavaRDDLike.mapToPair(JavaRDDLike.scala:45) | |
| at org.apache.hudi.client.common.HoodieSparkEngineContext.mapToPair(HoodieSparkEngineContext.java:89) | |
| at org.apache.hudi.common.fs.FSUtils.parallelizeSubPathProcess(FSUtils.java:661) | |
| at org.apache.hudi.common.fs.FSUtils.deleteDir(FSUtils.java:621) | |
| at org.apache.hudi.table.marker.DirectMarkerFiles.deleteMarkerDir(DirectMarkerFiles.java:74) | |
| at org.apache.hudi.table.marker.MarkerFiles.quietDeleteMarkerDir(MarkerFiles.java:89) | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.postCommit(AbstractHoodieWriteClient.java:434) | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.commitStats(AbstractHoodieWriteClient.java:191) | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.commitStats(AbstractHoodieWriteClient.java:171) | |
| at org.apache.hudi.internal.DataSourceInternalWriterHelper.commit(DataSourceInternalWriterHelper.java:83) | |
| ... 108 more | |
| Caused by: java.io.NotSerializableException: org.apache.hudi.common.fs.FSUtils$$Lambda$4224/1845791682 | |
| Serialization stack: | |
| - object not serializable (class: org.apache.hudi.common.fs.FSUtils$$Lambda$4224/1845791682, value: org.apache.hudi.common.fs.FSUtils$$Lambda$4224/1845791682@38cc8317) | |
| - element of array (index: 0) | |
| - array (class [Ljava.lang.Object;, size 2) | |
| - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;) | |
| - object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.hudi.common.fs.FSUtils, functionalInterfaceMethod=org/apache/hudi/common/function/SerializablePairFunction.call:(Ljava/lang/Object;)Lorg/apache/hudi/common/util/collection/Pair;, implementation=invokeStatic org/apache/hudi/common/fs/FSUtils.lambda$parallelizeSubPathProcess$a21e2c7$1:(Ljava/util/function/Function;Lorg/apache/hudi/common/config/SerializableConfiguration;Ljava/lang/String;)Lorg/apache/hudi/common/util/collection/Pair;, instantiatedMethodType=(Ljava/lang/String;)Lorg/apache/hudi/common/util/collection/Pair;, numCaptured=2]) | |
| - writeReplace data (class: java.lang.invoke.SerializedLambda) | |
| - object (class org.apache.hudi.common.fs.FSUtils$$Lambda$4226/2018545795, org.apache.hudi.common.fs.FSUtils$$Lambda$4226/2018545795@c022524) | |
| - element of array (index: 0) | |
| - array (class [Ljava.lang.Object;, size 1) | |
| - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;) | |
| - object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.hudi.client.common.HoodieSparkEngineContext, functionalInterfaceMethod=org/apache/spark/api/java/function/PairFunction.call:(Ljava/lang/Object;)Lscala/Tuple2;, implementation=invokeStatic org/apache/hudi/client/common/HoodieSparkEngineContext.lambda$mapToPair$786cea6a$1:(Lorg/apache/hudi/common/function/SerializablePairFunction;Ljava/lang/Object;)Lscala/Tuple2;, instantiatedMethodType=(Ljava/lang/Object;)Lscala/Tuple2;, numCaptured=1]) | |
| - writeReplace data (class: java.lang.invoke.SerializedLambda) | |
| - object (class org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$4227/1347279361, org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$4227/1347279361@441f827e) | |
| - element of array (index: 0) | |
| - array (class [Ljava.lang.Object;, size 1) | |
| - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;) | |
| - object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.spark.api.java.JavaPairRDD$, functionalInterfaceMethod=scala/Function1.apply:(Ljava/lang/Object;)Ljava/lang/Object;, implementation=invokeStatic org/apache/spark/api/java/JavaPairRDD$.$anonfun$pairFunToScalaFun$1:(Lorg/apache/spark/api/java/function/PairFunction;Ljava/lang/Object;)Lscala/Tuple2;, instantiatedMethodType=(Ljava/lang/Object;)Lscala/Tuple2;, numCaptured=1]) | |
| - writeReplace data (class: java.lang.invoke.SerializedLambda) | |
| - object (class org.apache.spark.api.java.JavaPairRDD$$$Lambda$4228/1967171315, org.apache.spark.api.java.JavaPairRDD$$$Lambda$4228/1967171315@6f8ac388) | |
| at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:41) | |
| at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:47) | |
| at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:101) | |
| at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:413) | |
| ... 128 more | |
| Suppressed: org.apache.hudi.exception.HoodieRollbackException: Failed to rollback s3a://siva-test-bucket-june-16/hudi_testing/hudi_3/ commits 20210806024337 | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.rollback(AbstractHoodieWriteClient.java:600) | |
| at org.apache.hudi.internal.DataSourceInternalWriterHelper.abort(DataSourceInternalWriterHelper.java:94) | |
| at org.apache.hudi.spark3.internal.HoodieDataSourceInternalBatchWrite.abort(HoodieDataSourceInternalBatchWrite.java:98) | |
| at org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2(WriteToDataSourceV2Exec.scala:403) | |
| at org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2$(WriteToDataSourceV2Exec.scala:361) | |
| at org.apache.spark.sql.execution.datasources.v2.AppendDataExec.writeWithV2(WriteToDataSourceV2Exec.scala:253) | |
| at org.apache.spark.sql.execution.datasources.v2.AppendDataExec.run(WriteToDataSourceV2Exec.scala:259) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result$lzycompute(V2CommandExec.scala:39) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result(V2CommandExec.scala:39) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.doExecute(V2CommandExec.scala:54) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121) | |
| at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:963) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) | |
| at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:963) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:354) | |
| at org.apache.hudi.HoodieSparkSqlWriter$.bulkInsertAsRow(HoodieSparkSqlWriter.scala:378) | |
| at org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:149) | |
| at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:165) | |
| at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:46) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:90) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121) | |
| at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:963) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) | |
| at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:963) | |
| at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:415) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:399) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:288) | |
| at $anonfun$res3$1(<console>:42) | |
| at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) | |
| at org.apache.spark.sql.SparkSession.time(SparkSession.scala:668) | |
| Caused by: org.apache.spark.SparkException: Task not serializable | |
| at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:416) | |
| at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:406) | |
| at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:162) | |
| at org.apache.spark.SparkContext.clean(SparkContext.scala:2362) | |
| at org.apache.spark.rdd.RDD.$anonfun$map$1(RDD.scala:396) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) | |
| at org.apache.spark.rdd.RDD.withScope(RDD.scala:388) | |
| at org.apache.spark.rdd.RDD.map(RDD.scala:395) | |
| at org.apache.spark.api.java.JavaRDDLike.mapToPair(JavaRDDLike.scala:117) | |
| at org.apache.spark.api.java.JavaRDDLike.mapToPair$(JavaRDDLike.scala:115) | |
| at org.apache.spark.api.java.AbstractJavaRDDLike.mapToPair(JavaRDDLike.scala:45) | |
| at org.apache.hudi.client.common.HoodieSparkEngineContext.mapToPair(HoodieSparkEngineContext.java:89) | |
| at org.apache.hudi.common.fs.FSUtils.parallelizeSubPathProcess(FSUtils.java:661) | |
| at org.apache.hudi.common.fs.FSUtils.deleteDir(FSUtils.java:621) | |
| at org.apache.hudi.table.marker.DirectMarkerFiles.deleteMarkerDir(DirectMarkerFiles.java:74) | |
| at org.apache.hudi.table.marker.MarkerFiles.quietDeleteMarkerDir(MarkerFiles.java:89) | |
| at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.execute(BaseRollbackActionExecutor.java:116) | |
| at org.apache.hudi.table.HoodieSparkCopyOnWriteTable.rollback(HoodieSparkCopyOnWriteTable.java:250) | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.rollback(AbstractHoodieWriteClient.java:589) | |
| ... 109 more | |
| Caused by: java.io.NotSerializableException: org.apache.hudi.common.fs.FSUtils$$Lambda$4224/1845791682 | |
| Serialization stack: | |
| - object not serializable (class: org.apache.hudi.common.fs.FSUtils$$Lambda$4224/1845791682, value: org.apache.hudi.common.fs.FSUtils$$Lambda$4224/1845791682@38cc8317) | |
| - element of array (index: 0) | |
| - array (class [Ljava.lang.Object;, size 2) | |
| - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;) | |
| - object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.hudi.common.fs.FSUtils, functionalInterfaceMethod=org/apache/hudi/common/function/SerializablePairFunction.call:(Ljava/lang/Object;)Lorg/apache/hudi/common/util/collection/Pair;, implementation=invokeStatic org/apache/hudi/common/fs/FSUtils.lambda$parallelizeSubPathProcess$a21e2c7$1:(Ljava/util/function/Function;Lorg/apache/hudi/common/config/SerializableConfiguration;Ljava/lang/String;)Lorg/apache/hudi/common/util/collection/Pair;, instantiatedMethodType=(Ljava/lang/String;)Lorg/apache/hudi/common/util/collection/Pair;, numCaptured=2]) | |
| - writeReplace data (class: java.lang.invoke.SerializedLambda) | |
| - object (class org.apache.hudi.common.fs.FSUtils$$Lambda$4226/2018545795, org.apache.hudi.common.fs.FSUtils$$Lambda$4226/2018545795@5ef7a514) | |
| - element of array (index: 0) | |
| - array (class [Ljava.lang.Object;, size 1) | |
| - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;) | |
| - object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.hudi.client.common.HoodieSparkEngineContext, functionalInterfaceMethod=org/apache/spark/api/java/function/PairFunction.call:(Ljava/lang/Object;)Lscala/Tuple2;, implementation=invokeStatic org/apache/hudi/client/common/HoodieSparkEngineContext.lambda$mapToPair$786cea6a$1:(Lorg/apache/hudi/common/function/SerializablePairFunction;Ljava/lang/Object;)Lscala/Tuple2;, instantiatedMethodType=(Ljava/lang/Object;)Lscala/Tuple2;, numCaptured=1]) | |
| - writeReplace data (class: java.lang.invoke.SerializedLambda) | |
| - object (class org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$4227/1347279361, org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$4227/1347279361@6b31241e) | |
| - element of array (index: 0) | |
| - array (class [Ljava.lang.Object;, size 1) | |
| - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;) | |
| - object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.spark.api.java.JavaPairRDD$, functionalInterfaceMethod=scala/Function1.apply:(Ljava/lang/Object;)Ljava/lang/Object;, implementation=invokeStatic org/apache/spark/api/java/JavaPairRDD$.$anonfun$pairFunToScalaFun$1:(Lorg/apache/spark/api/java/function/PairFunction;Ljava/lang/Object;)Lscala/Tuple2;, instantiatedMethodType=(Ljava/lang/Object;)Lscala/Tuple2;, numCaptured=1]) | |
| - writeReplace data (class: java.lang.invoke.SerializedLambda) | |
| - object (class org.apache.spark.api.java.JavaPairRDD$$$Lambda$4228/1967171315, org.apache.spark.api.java.JavaPairRDD$$$Lambda$4228/1967171315@66e9e248) | |
| at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:41) | |
| at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:47) | |
| at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:101) | |
| at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:413) | |
| ... 128 more |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment