Created
July 29, 2021 20:41
-
-
Save nsivabalan/0e1c4c4319ed15ac2af8aac865ca6926 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| > CREATE OR REPLACE TABLE delta_ext ( | |
| > public BOOLEAN, | |
| > repo_id BIGINT, | |
| > repo_name STRING, | |
| > repo_url STRING, | |
| > payload STRING, | |
| > created_at TIMESTAMP, | |
| > id STRING, | |
| > other STRING, | |
| > randomId DOUBLE, | |
| > date_col STRING, | |
| > type STRING) | |
| > USING DELTA location 's3a://siva-test-bucket-june-16/hudi_testing/gh_arch_dump/delta_2' | |
| > PARTITIONED BY (type); | |
| 21/07/29 13:51:39 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'. | |
| 21/07/29 13:51:39 ERROR Utils: Aborting task | |
| org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree: | |
| Exchange hashpartitioning(coalesce(add#165.path, remove#166.path), 50), false, [id=#104] | |
| +- *(1) Project [txn#164, add#165, remove#166, metaData#167, protocol#168, cdc#169, commitInfo#170, UDF(input_file_name()) AS file#172] | |
| +- *(1) SerializeFromObject [if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn)) null else named_struct(appId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn).appId, true, false), version, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn).version, lastUpdated, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn).lastUpdated)) AS txn#164, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add)) null else named_struct(path, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).path, true, false), partitionValues, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -1), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -1), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -2), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -2), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).partitionValues), size, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).size, modificationTime, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).modificationTime, dataChange, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).dataChange, stats, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).stats, true, false), tags, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -3), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -3), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -4), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -4), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).tags)) AS add#165, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove)) null else named_struct(path, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).path, true, false), deletionTimestamp, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).deletionTimestamp), dataChange, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).dataChange, extendedFileMetadata, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).extendedFileMetadata, partitionValues, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -5), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -5), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -6), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -6), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).partitionValues), size, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).size, tags, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -7), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -7), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -8), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -8), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).tags)) AS remove#166, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData)) null else named_struct(id, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).id, true, false), name, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).name, true, false), description, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).description, true, false), format, if (isnull(knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).format)) null else named_struct(provider, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).format).provider, true, false), options, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -9), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -9), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -10), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -10), true, false), knownnotnull(knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).format).options)), schemaString, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).schemaString, true, false), partitionColumns, mapobjects(lambdavariable(MapObject, ObjectType(class java.lang.String), true, -11), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(MapObject, ObjectType(class java.lang.String), true, -11), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).partitionColumns, None), configuration, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -12), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -12), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -13), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -13), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).configuration), createdTime, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).createdTime)) AS metaData#167, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).protocol)) null else named_struct(minReaderVersion, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).protocol).minReaderVersion, minWriterVersion, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).protocol).minWriterVersion) AS protocol#168, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc)) null else named_struct(path, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).path, true, false), partitionValues, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -14), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -14), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -15), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -15), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).partitionValues), size, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).size, tags, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -16), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -16), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -17), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -17), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).tags)) AS cdc#169, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo)) null else named_struct(version, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).version), timestamp, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, TimestampType, fromJavaTimestamp, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).timestamp, true, false), userId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).userId), true, false), userName, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).userName), true, false), operation, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).operation, true, false), operationParameters, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -18), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -18), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -19), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -19), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).operationParameters), job, if (isnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job))) null else named_struct(jobId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).jobId, true, false), jobName, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).jobName, true, false), runId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).runId, true, false), jobOwnerId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).jobOwnerId, true, false), triggerType, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).triggerType, true, false)), notebook, if (isnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.NotebookInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).notebook))) null else named_struct(notebookId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.NotebookInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).notebook)).notebookId, true, false)), clusterId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).clusterId), true, false), readVersion, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).readVersion), isolationLevel, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).isolationLevel), true, false), isBlindAppend, unwrapoption(BooleanType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).isBlindAppend), ... 4 more fields) AS commitInfo#170] | |
| +- MapPartitions org.apache.spark.sql.delta.Snapshot$$Lambda$3365/464171441@f9ad4c9, obj#163: org.apache.spark.sql.delta.actions.SingleAction | |
| +- DeserializeToObject newInstance(class org.apache.spark.sql.delta.actions.SingleAction), obj#162: org.apache.spark.sql.delta.actions.SingleAction | |
| +- FileScan json [txn#126,add#127,remove#128,metaData#129,protocol#130,cdc#131,commitInfo#132] Batched: false, DataFilters: [], Format: JSON, Location: DeltaLogFileIndex[s3a://siva-test-bucket-june-16/hudi_testing/gh_arch_dump/delta_2/_delta_log/000..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<txn:struct<appId:string,version:bigint,lastUpdated:bigint>,add:struct<path:string,partitio... | |
| at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(ShuffleExchangeExec.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDD(WholeStageCodegenExec.scala:525) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(WholeStageCodegenExec.scala:453) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(WholeStageCodegenExec.scala:452) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:496) | |
| at org.apache.spark.sql.execution.SortExec.inputRDDs(SortExec.scala:132) | |
| at org.apache.spark.sql.execution.ProjectExec.inputRDDs(basicPhysicalOperators.scala:47) | |
| at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.DeserializeToObjectExec.doExecute(objects.scala:96) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.MapPartitionsExec.doExecute(objects.scala:192) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDD(WholeStageCodegenExec.scala:525) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(WholeStageCodegenExec.scala:453) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(WholeStageCodegenExec.scala:452) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:496) | |
| at org.apache.spark.sql.execution.SerializeFromObjectExec.inputRDDs(objects.scala:117) | |
| at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121) | |
| at org.apache.spark.sql.delta.util.StateCache$CachedDS.<init>(StateCache.scala:52) | |
| at org.apache.spark.sql.delta.util.StateCache.cacheDS(StateCache.scala:100) | |
| at org.apache.spark.sql.delta.util.StateCache.cacheDS$(StateCache.scala:99) | |
| at org.apache.spark.sql.delta.Snapshot.cacheDS(Snapshot.scala:55) | |
| at org.apache.spark.sql.delta.Snapshot.cachedState$lzycompute(Snapshot.scala:123) | |
| at org.apache.spark.sql.delta.Snapshot.cachedState(Snapshot.scala:122) | |
| at org.apache.spark.sql.delta.Snapshot.state(Snapshot.scala:126) | |
| at org.apache.spark.sql.delta.Snapshot.$anonfun$computedState$1(Snapshot.scala:146) | |
| at org.apache.spark.sql.delta.util.DeltaProgressReporter.withJobDescription(DeltaProgressReporter.scala:53) | |
| at org.apache.spark.sql.delta.util.DeltaProgressReporter.withStatusCode(DeltaProgressReporter.scala:32) | |
| at org.apache.spark.sql.delta.util.DeltaProgressReporter.withStatusCode$(DeltaProgressReporter.scala:27) | |
| at org.apache.spark.sql.delta.Snapshot.withStatusCode(Snapshot.scala:55) | |
| at org.apache.spark.sql.delta.Snapshot.computedState$lzycompute(Snapshot.scala:143) | |
| at org.apache.spark.sql.delta.Snapshot.computedState(Snapshot.scala:142) | |
| at org.apache.spark.sql.delta.Snapshot.protocol(Snapshot.scala:184) | |
| at org.apache.spark.sql.delta.Snapshot.init(Snapshot.scala:82) | |
| at org.apache.spark.sql.delta.Snapshot.<init>(Snapshot.scala:300) | |
| at org.apache.spark.sql.delta.SnapshotManagement.createSnapshot(SnapshotManagement.scala:221) | |
| at org.apache.spark.sql.delta.SnapshotManagement.createSnapshot$(SnapshotManagement.scala:209) | |
| at org.apache.spark.sql.delta.DeltaLog.createSnapshot(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.SnapshotManagement.$anonfun$updateInternal$1(SnapshotManagement.scala:301) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.DeltaLog.recordOperation(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.DeltaLog.recordDeltaOperation(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.SnapshotManagement.updateInternal(SnapshotManagement.scala:285) | |
| at org.apache.spark.sql.delta.SnapshotManagement.updateInternal$(SnapshotManagement.scala:284) | |
| at org.apache.spark.sql.delta.DeltaLog.updateInternal(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.SnapshotManagement.$anonfun$update$1(SnapshotManagement.scala:246) | |
| at org.apache.spark.sql.delta.DeltaLog.lockInterruptibly(DeltaLog.scala:153) | |
| at org.apache.spark.sql.delta.SnapshotManagement.update(SnapshotManagement.scala:246) | |
| at org.apache.spark.sql.delta.SnapshotManagement.update$(SnapshotManagement.scala:242) | |
| at org.apache.spark.sql.delta.DeltaLog.update(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommit(OptimisticTransaction.scala:637) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommit$(OptimisticTransaction.scala:606) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.doCommit(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.$anonfun$doCommitRetryIteratively$2(OptimisticTransaction.scala:578) | |
| at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordDeltaOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.$anonfun$doCommitRetryIteratively$1(OptimisticTransaction.scala:574) | |
| at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23) | |
| at org.apache.spark.sql.delta.DeltaLog.lockInterruptibly(DeltaLog.scala:153) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommitRetryIteratively(OptimisticTransaction.scala:569) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommitRetryIteratively$(OptimisticTransaction.scala:566) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.doCommitRetryIteratively(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.$anonfun$commit$1(OptimisticTransaction.scala:439) | |
| at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordDeltaOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.commit(OptimisticTransaction.scala:390) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.commit$(OptimisticTransaction.scala:388) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.commit(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.createTransactionLogOrVerify$1(CreateDeltaTableCommand.scala:175) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.$anonfun$run$2(CreateDeltaTableCommand.scala:191) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.recordOperation(CreateDeltaTableCommand.scala:48) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.recordDeltaOperation(CreateDeltaTableCommand.scala:48) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.run(CreateDeltaTableCommand.scala:101) | |
| at org.apache.spark.sql.delta.catalog.DeltaCatalog.org$apache$spark$sql$delta$catalog$DeltaCatalog$$createDeltaTable(DeltaCatalog.scala:140) | |
| at org.apache.spark.sql.delta.catalog.DeltaCatalog$StagedDeltaTableV2.commitStagedChanges(DeltaCatalog.scala:328) | |
| at org.apache.spark.sql.execution.datasources.v2.AtomicReplaceTableExec.$anonfun$commitOrAbortStagedChanges$1(ReplaceTableExec.scala:82) | |
| at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1411) | |
| at org.apache.spark.sql.execution.datasources.v2.AtomicReplaceTableExec.commitOrAbortStagedChanges(ReplaceTableExec.scala:81) | |
| at org.apache.spark.sql.execution.datasources.v2.AtomicReplaceTableExec.run(ReplaceTableExec.scala:74) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result$lzycompute(V2CommandExec.scala:39) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result(V2CommandExec.scala:39) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.executeCollect(V2CommandExec.scala:45) | |
| at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:229) | |
| at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3618) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) | |
| at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3616) | |
| at org.apache.spark.sql.Dataset.<init>(Dataset.scala:229) | |
| at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:100) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97) | |
| at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:607) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:602) | |
| at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:63) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:377) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.$anonfun$processLine$1(SparkSQLCLIDriver.scala:496) | |
| at scala.collection.Iterator.foreach(Iterator.scala:941) | |
| at scala.collection.Iterator.foreach$(Iterator.scala:941) | |
| at scala.collection.AbstractIterator.foreach(Iterator.scala:1429) | |
| at scala.collection.IterableLike.foreach(IterableLike.scala:74) | |
| at scala.collection.IterableLike.foreach$(IterableLike.scala:73) | |
| at scala.collection.AbstractIterable.foreach(Iterable.scala:56) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processLine(SparkSQLCLIDriver.scala:490) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:282) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) | |
| at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:928) | |
| at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) | |
| at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) | |
| at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) | |
| at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1007) | |
| at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1016) | |
| at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) | |
| Caused by: java.lang.IllegalArgumentException: Compression codec com.hadoop.compression.lzo.LzoCodec not found. | |
| at org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses(CompressionCodecFactory.java:139) | |
| at org.apache.hadoop.io.compress.CompressionCodecFactory.<init>(CompressionCodecFactory.java:180) | |
| at org.apache.spark.sql.execution.datasources.TextBasedFileFormat.isSplitable(FileFormat.scala:179) | |
| at org.apache.spark.sql.execution.datasources.json.JsonFileFormat.isSplitable(JsonFileFormat.scala:49) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.$anonfun$createNonBucketedReadRDD$3(DataSourceScanExec.scala:575) | |
| at scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:245) | |
| at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) | |
| at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) | |
| at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38) | |
| at scala.collection.TraversableLike.flatMap(TraversableLike.scala:245) | |
| at scala.collection.TraversableLike.flatMap$(TraversableLike.scala:242) | |
| at scala.collection.AbstractTraversable.flatMap(Traversable.scala:108) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.$anonfun$createNonBucketedReadRDD$2(DataSourceScanExec.scala:571) | |
| at scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:245) | |
| at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) | |
| at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) | |
| at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198) | |
| at scala.collection.TraversableLike.flatMap(TraversableLike.scala:245) | |
| at scala.collection.TraversableLike.flatMap$(TraversableLike.scala:242) | |
| at scala.collection.mutable.ArrayOps$ofRef.flatMap(ArrayOps.scala:198) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.createNonBucketedReadRDD(DataSourceScanExec.scala:570) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:405) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:390) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.doExecute(DataSourceScanExec.scala:473) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.DeserializeToObjectExec.doExecute(objects.scala:96) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.MapPartitionsExec.doExecute(objects.scala:192) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDD(WholeStageCodegenExec.scala:525) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(WholeStageCodegenExec.scala:453) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(WholeStageCodegenExec.scala:452) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:496) | |
| at org.apache.spark.sql.execution.SerializeFromObjectExec.inputRDDs(objects.scala:117) | |
| at org.apache.spark.sql.execution.ProjectExec.inputRDDs(basicPhysicalOperators.scala:47) | |
| at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD$lzycompute(ShuffleExchangeExec.scala:106) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD(ShuffleExchangeExec.scala:106) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency$lzycompute(ShuffleExchangeExec.scala:139) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency(ShuffleExchangeExec.scala:137) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.$anonfun$doExecute$1(ShuffleExchangeExec.scala:154) | |
| at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52) | |
| ... 165 more | |
| Caused by: java.lang.ClassNotFoundException: Class com.hadoop.compression.lzo.LzoCodec not found | |
| at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2499) | |
| at org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses(CompressionCodecFactory.java:132) | |
| ... 223 more | |
| 21/07/29 13:51:39 ERROR SparkSQLDriver: Failed in [CREATE OR REPLACE TABLE delta_ext ( | |
| public BOOLEAN, | |
| repo_id BIGINT, | |
| repo_name STRING, | |
| repo_url STRING, | |
| payload STRING, | |
| created_at TIMESTAMP, | |
| id STRING, | |
| other STRING, | |
| randomId DOUBLE, | |
| date_col STRING, | |
| type STRING) | |
| USING DELTA location 's3a://siva-test-bucket-june-16/hudi_testing/gh_arch_dump/delta_2' | |
| PARTITIONED BY (type)] | |
| org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree: | |
| Exchange hashpartitioning(coalesce(add#165.path, remove#166.path), 50), false, [id=#104] | |
| +- *(1) Project [txn#164, add#165, remove#166, metaData#167, protocol#168, cdc#169, commitInfo#170, UDF(input_file_name()) AS file#172] | |
| +- *(1) SerializeFromObject [if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn)) null else named_struct(appId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn).appId, true, false), version, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn).version, lastUpdated, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn).lastUpdated)) AS txn#164, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add)) null else named_struct(path, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).path, true, false), partitionValues, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -1), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -1), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -2), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -2), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).partitionValues), size, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).size, modificationTime, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).modificationTime, dataChange, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).dataChange, stats, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).stats, true, false), tags, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -3), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -3), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -4), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -4), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).tags)) AS add#165, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove)) null else named_struct(path, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).path, true, false), deletionTimestamp, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).deletionTimestamp), dataChange, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).dataChange, extendedFileMetadata, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).extendedFileMetadata, partitionValues, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -5), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -5), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -6), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -6), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).partitionValues), size, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).size, tags, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -7), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -7), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -8), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -8), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).tags)) AS remove#166, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData)) null else named_struct(id, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).id, true, false), name, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).name, true, false), description, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).description, true, false), format, if (isnull(knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).format)) null else named_struct(provider, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).format).provider, true, false), options, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -9), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -9), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -10), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -10), true, false), knownnotnull(knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).format).options)), schemaString, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).schemaString, true, false), partitionColumns, mapobjects(lambdavariable(MapObject, ObjectType(class java.lang.String), true, -11), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(MapObject, ObjectType(class java.lang.String), true, -11), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).partitionColumns, None), configuration, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -12), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -12), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -13), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -13), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).configuration), createdTime, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).createdTime)) AS metaData#167, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).protocol)) null else named_struct(minReaderVersion, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).protocol).minReaderVersion, minWriterVersion, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).protocol).minWriterVersion) AS protocol#168, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc)) null else named_struct(path, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).path, true, false), partitionValues, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -14), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -14), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -15), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -15), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).partitionValues), size, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).size, tags, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -16), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -16), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -17), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -17), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).tags)) AS cdc#169, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo)) null else named_struct(version, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).version), timestamp, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, TimestampType, fromJavaTimestamp, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).timestamp, true, false), userId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).userId), true, false), userName, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).userName), true, false), operation, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).operation, true, false), operationParameters, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -18), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -18), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -19), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -19), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).operationParameters), job, if (isnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job))) null else named_struct(jobId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).jobId, true, false), jobName, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).jobName, true, false), runId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).runId, true, false), jobOwnerId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).jobOwnerId, true, false), triggerType, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).triggerType, true, false)), notebook, if (isnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.NotebookInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).notebook))) null else named_struct(notebookId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.NotebookInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).notebook)).notebookId, true, false)), clusterId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).clusterId), true, false), readVersion, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).readVersion), isolationLevel, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).isolationLevel), true, false), isBlindAppend, unwrapoption(BooleanType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).isBlindAppend), ... 4 more fields) AS commitInfo#170] | |
| +- MapPartitions org.apache.spark.sql.delta.Snapshot$$Lambda$3365/464171441@f9ad4c9, obj#163: org.apache.spark.sql.delta.actions.SingleAction | |
| +- DeserializeToObject newInstance(class org.apache.spark.sql.delta.actions.SingleAction), obj#162: org.apache.spark.sql.delta.actions.SingleAction | |
| +- FileScan json [txn#126,add#127,remove#128,metaData#129,protocol#130,cdc#131,commitInfo#132] Batched: false, DataFilters: [], Format: JSON, Location: DeltaLogFileIndex[s3a://siva-test-bucket-june-16/hudi_testing/gh_arch_dump/delta_2/_delta_log/000..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<txn:struct<appId:string,version:bigint,lastUpdated:bigint>,add:struct<path:string,partitio... | |
| at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(ShuffleExchangeExec.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDD(WholeStageCodegenExec.scala:525) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(WholeStageCodegenExec.scala:453) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(WholeStageCodegenExec.scala:452) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:496) | |
| at org.apache.spark.sql.execution.SortExec.inputRDDs(SortExec.scala:132) | |
| at org.apache.spark.sql.execution.ProjectExec.inputRDDs(basicPhysicalOperators.scala:47) | |
| at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.DeserializeToObjectExec.doExecute(objects.scala:96) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.MapPartitionsExec.doExecute(objects.scala:192) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDD(WholeStageCodegenExec.scala:525) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(WholeStageCodegenExec.scala:453) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(WholeStageCodegenExec.scala:452) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:496) | |
| at org.apache.spark.sql.execution.SerializeFromObjectExec.inputRDDs(objects.scala:117) | |
| at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121) | |
| at org.apache.spark.sql.delta.util.StateCache$CachedDS.<init>(StateCache.scala:52) | |
| at org.apache.spark.sql.delta.util.StateCache.cacheDS(StateCache.scala:100) | |
| at org.apache.spark.sql.delta.util.StateCache.cacheDS$(StateCache.scala:99) | |
| at org.apache.spark.sql.delta.Snapshot.cacheDS(Snapshot.scala:55) | |
| at org.apache.spark.sql.delta.Snapshot.cachedState$lzycompute(Snapshot.scala:123) | |
| at org.apache.spark.sql.delta.Snapshot.cachedState(Snapshot.scala:122) | |
| at org.apache.spark.sql.delta.Snapshot.state(Snapshot.scala:126) | |
| at org.apache.spark.sql.delta.Snapshot.$anonfun$computedState$1(Snapshot.scala:146) | |
| at org.apache.spark.sql.delta.util.DeltaProgressReporter.withJobDescription(DeltaProgressReporter.scala:53) | |
| at org.apache.spark.sql.delta.util.DeltaProgressReporter.withStatusCode(DeltaProgressReporter.scala:32) | |
| at org.apache.spark.sql.delta.util.DeltaProgressReporter.withStatusCode$(DeltaProgressReporter.scala:27) | |
| at org.apache.spark.sql.delta.Snapshot.withStatusCode(Snapshot.scala:55) | |
| at org.apache.spark.sql.delta.Snapshot.computedState$lzycompute(Snapshot.scala:143) | |
| at org.apache.spark.sql.delta.Snapshot.computedState(Snapshot.scala:142) | |
| at org.apache.spark.sql.delta.Snapshot.protocol(Snapshot.scala:184) | |
| at org.apache.spark.sql.delta.Snapshot.init(Snapshot.scala:82) | |
| at org.apache.spark.sql.delta.Snapshot.<init>(Snapshot.scala:300) | |
| at org.apache.spark.sql.delta.SnapshotManagement.createSnapshot(SnapshotManagement.scala:221) | |
| at org.apache.spark.sql.delta.SnapshotManagement.createSnapshot$(SnapshotManagement.scala:209) | |
| at org.apache.spark.sql.delta.DeltaLog.createSnapshot(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.SnapshotManagement.$anonfun$updateInternal$1(SnapshotManagement.scala:301) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.DeltaLog.recordOperation(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.DeltaLog.recordDeltaOperation(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.SnapshotManagement.updateInternal(SnapshotManagement.scala:285) | |
| at org.apache.spark.sql.delta.SnapshotManagement.updateInternal$(SnapshotManagement.scala:284) | |
| at org.apache.spark.sql.delta.DeltaLog.updateInternal(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.SnapshotManagement.$anonfun$update$1(SnapshotManagement.scala:246) | |
| at org.apache.spark.sql.delta.DeltaLog.lockInterruptibly(DeltaLog.scala:153) | |
| at org.apache.spark.sql.delta.SnapshotManagement.update(SnapshotManagement.scala:246) | |
| at org.apache.spark.sql.delta.SnapshotManagement.update$(SnapshotManagement.scala:242) | |
| at org.apache.spark.sql.delta.DeltaLog.update(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommit(OptimisticTransaction.scala:637) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommit$(OptimisticTransaction.scala:606) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.doCommit(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.$anonfun$doCommitRetryIteratively$2(OptimisticTransaction.scala:578) | |
| at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordDeltaOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.$anonfun$doCommitRetryIteratively$1(OptimisticTransaction.scala:574) | |
| at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23) | |
| at org.apache.spark.sql.delta.DeltaLog.lockInterruptibly(DeltaLog.scala:153) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommitRetryIteratively(OptimisticTransaction.scala:569) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommitRetryIteratively$(OptimisticTransaction.scala:566) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.doCommitRetryIteratively(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.$anonfun$commit$1(OptimisticTransaction.scala:439) | |
| at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordDeltaOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.commit(OptimisticTransaction.scala:390) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.commit$(OptimisticTransaction.scala:388) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.commit(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.createTransactionLogOrVerify$1(CreateDeltaTableCommand.scala:175) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.$anonfun$run$2(CreateDeltaTableCommand.scala:191) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.recordOperation(CreateDeltaTableCommand.scala:48) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.recordDeltaOperation(CreateDeltaTableCommand.scala:48) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.run(CreateDeltaTableCommand.scala:101) | |
| at org.apache.spark.sql.delta.catalog.DeltaCatalog.org$apache$spark$sql$delta$catalog$DeltaCatalog$$createDeltaTable(DeltaCatalog.scala:140) | |
| at org.apache.spark.sql.delta.catalog.DeltaCatalog$StagedDeltaTableV2.commitStagedChanges(DeltaCatalog.scala:328) | |
| at org.apache.spark.sql.execution.datasources.v2.AtomicReplaceTableExec.$anonfun$commitOrAbortStagedChanges$1(ReplaceTableExec.scala:82) | |
| at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1411) | |
| at org.apache.spark.sql.execution.datasources.v2.AtomicReplaceTableExec.commitOrAbortStagedChanges(ReplaceTableExec.scala:81) | |
| at org.apache.spark.sql.execution.datasources.v2.AtomicReplaceTableExec.run(ReplaceTableExec.scala:74) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result$lzycompute(V2CommandExec.scala:39) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result(V2CommandExec.scala:39) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.executeCollect(V2CommandExec.scala:45) | |
| at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:229) | |
| at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3618) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) | |
| at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3616) | |
| at org.apache.spark.sql.Dataset.<init>(Dataset.scala:229) | |
| at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:100) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97) | |
| at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:607) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:602) | |
| at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:63) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:377) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.$anonfun$processLine$1(SparkSQLCLIDriver.scala:496) | |
| at scala.collection.Iterator.foreach(Iterator.scala:941) | |
| at scala.collection.Iterator.foreach$(Iterator.scala:941) | |
| at scala.collection.AbstractIterator.foreach(Iterator.scala:1429) | |
| at scala.collection.IterableLike.foreach(IterableLike.scala:74) | |
| at scala.collection.IterableLike.foreach$(IterableLike.scala:73) | |
| at scala.collection.AbstractIterable.foreach(Iterable.scala:56) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processLine(SparkSQLCLIDriver.scala:490) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:282) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) | |
| at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:928) | |
| at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) | |
| at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) | |
| at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) | |
| at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1007) | |
| at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1016) | |
| at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) | |
| Caused by: java.lang.IllegalArgumentException: Compression codec com.hadoop.compression.lzo.LzoCodec not found. | |
| at org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses(CompressionCodecFactory.java:139) | |
| at org.apache.hadoop.io.compress.CompressionCodecFactory.<init>(CompressionCodecFactory.java:180) | |
| at org.apache.spark.sql.execution.datasources.TextBasedFileFormat.isSplitable(FileFormat.scala:179) | |
| at org.apache.spark.sql.execution.datasources.json.JsonFileFormat.isSplitable(JsonFileFormat.scala:49) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.$anonfun$createNonBucketedReadRDD$3(DataSourceScanExec.scala:575) | |
| at scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:245) | |
| at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) | |
| at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) | |
| at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38) | |
| at scala.collection.TraversableLike.flatMap(TraversableLike.scala:245) | |
| at scala.collection.TraversableLike.flatMap$(TraversableLike.scala:242) | |
| at scala.collection.AbstractTraversable.flatMap(Traversable.scala:108) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.$anonfun$createNonBucketedReadRDD$2(DataSourceScanExec.scala:571) | |
| at scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:245) | |
| at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) | |
| at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) | |
| at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198) | |
| at scala.collection.TraversableLike.flatMap(TraversableLike.scala:245) | |
| at scala.collection.TraversableLike.flatMap$(TraversableLike.scala:242) | |
| at scala.collection.mutable.ArrayOps$ofRef.flatMap(ArrayOps.scala:198) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.createNonBucketedReadRDD(DataSourceScanExec.scala:570) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:405) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:390) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.doExecute(DataSourceScanExec.scala:473) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.DeserializeToObjectExec.doExecute(objects.scala:96) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.MapPartitionsExec.doExecute(objects.scala:192) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDD(WholeStageCodegenExec.scala:525) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(WholeStageCodegenExec.scala:453) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(WholeStageCodegenExec.scala:452) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:496) | |
| at org.apache.spark.sql.execution.SerializeFromObjectExec.inputRDDs(objects.scala:117) | |
| at org.apache.spark.sql.execution.ProjectExec.inputRDDs(basicPhysicalOperators.scala:47) | |
| at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD$lzycompute(ShuffleExchangeExec.scala:106) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD(ShuffleExchangeExec.scala:106) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency$lzycompute(ShuffleExchangeExec.scala:139) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency(ShuffleExchangeExec.scala:137) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.$anonfun$doExecute$1(ShuffleExchangeExec.scala:154) | |
| at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52) | |
| ... 165 more | |
| Caused by: java.lang.ClassNotFoundException: Class com.hadoop.compression.lzo.LzoCodec not found | |
| at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2499) | |
| at org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses(CompressionCodecFactory.java:132) | |
| ... 223 more | |
| org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree: | |
| Exchange hashpartitioning(coalesce(add#165.path, remove#166.path), 50), false, [id=#104] | |
| +- *(1) Project [txn#164, add#165, remove#166, metaData#167, protocol#168, cdc#169, commitInfo#170, UDF(input_file_name()) AS file#172] | |
| +- *(1) SerializeFromObject [if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn)) null else named_struct(appId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn).appId, true, false), version, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn).version, lastUpdated, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).txn).lastUpdated)) AS txn#164, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add)) null else named_struct(path, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).path, true, false), partitionValues, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -1), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -1), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -2), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -2), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).partitionValues), size, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).size, modificationTime, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).modificationTime, dataChange, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).dataChange, stats, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).stats, true, false), tags, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -3), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -3), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -4), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -4), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).add).tags)) AS add#165, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove)) null else named_struct(path, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).path, true, false), deletionTimestamp, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).deletionTimestamp), dataChange, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).dataChange, extendedFileMetadata, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).extendedFileMetadata, partitionValues, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -5), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -5), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -6), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -6), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).partitionValues), size, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).size, tags, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -7), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -7), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -8), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -8), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).remove).tags)) AS remove#166, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData)) null else named_struct(id, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).id, true, false), name, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).name, true, false), description, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).description, true, false), format, if (isnull(knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).format)) null else named_struct(provider, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).format).provider, true, false), options, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -9), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -9), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -10), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -10), true, false), knownnotnull(knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).format).options)), schemaString, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).schemaString, true, false), partitionColumns, mapobjects(lambdavariable(MapObject, ObjectType(class java.lang.String), true, -11), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(MapObject, ObjectType(class java.lang.String), true, -11), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).partitionColumns, None), configuration, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -12), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -12), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -13), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -13), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).configuration), createdTime, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).metaData).createdTime)) AS metaData#167, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).protocol)) null else named_struct(minReaderVersion, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).protocol).minReaderVersion, minWriterVersion, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).protocol).minWriterVersion) AS protocol#168, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc)) null else named_struct(path, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).path, true, false), partitionValues, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -14), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -14), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -15), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -15), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).partitionValues), size, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).size, tags, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -16), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -16), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -17), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -17), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).cdc).tags)) AS cdc#169, if (isnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo)) null else named_struct(version, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).version), timestamp, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, TimestampType, fromJavaTimestamp, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).timestamp, true, false), userId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).userId), true, false), userName, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).userName), true, false), operation, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).operation, true, false), operationParameters, externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -18), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key, ObjectType(class java.lang.String), true, -18), true, false), lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -19), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_value, ObjectType(class java.lang.String), true, -19), true, false), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).operationParameters), job, if (isnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job))) null else named_struct(jobId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).jobId, true, false), jobName, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).jobName, true, false), runId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).runId, true, false), jobOwnerId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).jobOwnerId, true, false), triggerType, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.JobInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).job)).triggerType, true, false)), notebook, if (isnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.NotebookInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).notebook))) null else named_struct(notebookId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(unwrapoption(ObjectType(class org.apache.spark.sql.delta.actions.NotebookInfo), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).notebook)).notebookId, true, false)), clusterId, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).clusterId), true, false), readVersion, unwrapoption(LongType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).readVersion), isolationLevel, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, unwrapoption(ObjectType(class java.lang.String), knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).isolationLevel), true, false), isBlindAppend, unwrapoption(BooleanType, knownnotnull(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.actions.SingleAction, true])).commitInfo).isBlindAppend), ... 4 more fields) AS commitInfo#170] | |
| +- MapPartitions org.apache.spark.sql.delta.Snapshot$$Lambda$3365/464171441@f9ad4c9, obj#163: org.apache.spark.sql.delta.actions.SingleAction | |
| +- DeserializeToObject newInstance(class org.apache.spark.sql.delta.actions.SingleAction), obj#162: org.apache.spark.sql.delta.actions.SingleAction | |
| +- FileScan json [txn#126,add#127,remove#128,metaData#129,protocol#130,cdc#131,commitInfo#132] Batched: false, DataFilters: [], Format: JSON, Location: DeltaLogFileIndex[s3a://siva-test-bucket-june-16/hudi_testing/gh_arch_dump/delta_2/_delta_log/000..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<txn:struct<appId:string,version:bigint,lastUpdated:bigint>,add:struct<path:string,partitio... | |
| at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(ShuffleExchangeExec.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDD(WholeStageCodegenExec.scala:525) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(WholeStageCodegenExec.scala:453) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(WholeStageCodegenExec.scala:452) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:496) | |
| at org.apache.spark.sql.execution.SortExec.inputRDDs(SortExec.scala:132) | |
| at org.apache.spark.sql.execution.ProjectExec.inputRDDs(basicPhysicalOperators.scala:47) | |
| at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.DeserializeToObjectExec.doExecute(objects.scala:96) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.MapPartitionsExec.doExecute(objects.scala:192) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDD(WholeStageCodegenExec.scala:525) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(WholeStageCodegenExec.scala:453) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(WholeStageCodegenExec.scala:452) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:496) | |
| at org.apache.spark.sql.execution.SerializeFromObjectExec.inputRDDs(objects.scala:117) | |
| at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121) | |
| at org.apache.spark.sql.delta.util.StateCache$CachedDS.<init>(StateCache.scala:52) | |
| at org.apache.spark.sql.delta.util.StateCache.cacheDS(StateCache.scala:100) | |
| at org.apache.spark.sql.delta.util.StateCache.cacheDS$(StateCache.scala:99) | |
| at org.apache.spark.sql.delta.Snapshot.cacheDS(Snapshot.scala:55) | |
| at org.apache.spark.sql.delta.Snapshot.cachedState$lzycompute(Snapshot.scala:123) | |
| at org.apache.spark.sql.delta.Snapshot.cachedState(Snapshot.scala:122) | |
| at org.apache.spark.sql.delta.Snapshot.state(Snapshot.scala:126) | |
| at org.apache.spark.sql.delta.Snapshot.$anonfun$computedState$1(Snapshot.scala:146) | |
| at org.apache.spark.sql.delta.util.DeltaProgressReporter.withJobDescription(DeltaProgressReporter.scala:53) | |
| at org.apache.spark.sql.delta.util.DeltaProgressReporter.withStatusCode(DeltaProgressReporter.scala:32) | |
| at org.apache.spark.sql.delta.util.DeltaProgressReporter.withStatusCode$(DeltaProgressReporter.scala:27) | |
| at org.apache.spark.sql.delta.Snapshot.withStatusCode(Snapshot.scala:55) | |
| at org.apache.spark.sql.delta.Snapshot.computedState$lzycompute(Snapshot.scala:143) | |
| at org.apache.spark.sql.delta.Snapshot.computedState(Snapshot.scala:142) | |
| at org.apache.spark.sql.delta.Snapshot.protocol(Snapshot.scala:184) | |
| at org.apache.spark.sql.delta.Snapshot.init(Snapshot.scala:82) | |
| at org.apache.spark.sql.delta.Snapshot.<init>(Snapshot.scala:300) | |
| at org.apache.spark.sql.delta.SnapshotManagement.createSnapshot(SnapshotManagement.scala:221) | |
| at org.apache.spark.sql.delta.SnapshotManagement.createSnapshot$(SnapshotManagement.scala:209) | |
| at org.apache.spark.sql.delta.DeltaLog.createSnapshot(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.SnapshotManagement.$anonfun$updateInternal$1(SnapshotManagement.scala:301) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.DeltaLog.recordOperation(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.DeltaLog.recordDeltaOperation(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.SnapshotManagement.updateInternal(SnapshotManagement.scala:285) | |
| at org.apache.spark.sql.delta.SnapshotManagement.updateInternal$(SnapshotManagement.scala:284) | |
| at org.apache.spark.sql.delta.DeltaLog.updateInternal(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.SnapshotManagement.$anonfun$update$1(SnapshotManagement.scala:246) | |
| at org.apache.spark.sql.delta.DeltaLog.lockInterruptibly(DeltaLog.scala:153) | |
| at org.apache.spark.sql.delta.SnapshotManagement.update(SnapshotManagement.scala:246) | |
| at org.apache.spark.sql.delta.SnapshotManagement.update$(SnapshotManagement.scala:242) | |
| at org.apache.spark.sql.delta.DeltaLog.update(DeltaLog.scala:60) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommit(OptimisticTransaction.scala:637) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommit$(OptimisticTransaction.scala:606) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.doCommit(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.$anonfun$doCommitRetryIteratively$2(OptimisticTransaction.scala:578) | |
| at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordDeltaOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.$anonfun$doCommitRetryIteratively$1(OptimisticTransaction.scala:574) | |
| at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23) | |
| at org.apache.spark.sql.delta.DeltaLog.lockInterruptibly(DeltaLog.scala:153) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommitRetryIteratively(OptimisticTransaction.scala:569) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.doCommitRetryIteratively$(OptimisticTransaction.scala:566) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.doCommitRetryIteratively(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.$anonfun$commit$1(OptimisticTransaction.scala:439) | |
| at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.recordDeltaOperation(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.commit(OptimisticTransaction.scala:390) | |
| at org.apache.spark.sql.delta.OptimisticTransactionImpl.commit$(OptimisticTransaction.scala:388) | |
| at org.apache.spark.sql.delta.OptimisticTransaction.commit(OptimisticTransaction.scala:81) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.createTransactionLogOrVerify$1(CreateDeltaTableCommand.scala:175) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.$anonfun$run$2(CreateDeltaTableCommand.scala:191) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77) | |
| at com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.recordOperation(CreateDeltaTableCommand.scala:48) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:103) | |
| at org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:89) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.recordDeltaOperation(CreateDeltaTableCommand.scala:48) | |
| at org.apache.spark.sql.delta.commands.CreateDeltaTableCommand.run(CreateDeltaTableCommand.scala:101) | |
| at org.apache.spark.sql.delta.catalog.DeltaCatalog.org$apache$spark$sql$delta$catalog$DeltaCatalog$$createDeltaTable(DeltaCatalog.scala:140) | |
| at org.apache.spark.sql.delta.catalog.DeltaCatalog$StagedDeltaTableV2.commitStagedChanges(DeltaCatalog.scala:328) | |
| at org.apache.spark.sql.execution.datasources.v2.AtomicReplaceTableExec.$anonfun$commitOrAbortStagedChanges$1(ReplaceTableExec.scala:82) | |
| at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1411) | |
| at org.apache.spark.sql.execution.datasources.v2.AtomicReplaceTableExec.commitOrAbortStagedChanges(ReplaceTableExec.scala:81) | |
| at org.apache.spark.sql.execution.datasources.v2.AtomicReplaceTableExec.run(ReplaceTableExec.scala:74) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result$lzycompute(V2CommandExec.scala:39) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result(V2CommandExec.scala:39) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.executeCollect(V2CommandExec.scala:45) | |
| at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:229) | |
| at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3618) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) | |
| at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3616) | |
| at org.apache.spark.sql.Dataset.<init>(Dataset.scala:229) | |
| at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:100) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97) | |
| at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:607) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764) | |
| at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:602) | |
| at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:63) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:377) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.$anonfun$processLine$1(SparkSQLCLIDriver.scala:496) | |
| at scala.collection.Iterator.foreach(Iterator.scala:941) | |
| at scala.collection.Iterator.foreach$(Iterator.scala:941) | |
| at scala.collection.AbstractIterator.foreach(Iterator.scala:1429) | |
| at scala.collection.IterableLike.foreach(IterableLike.scala:74) | |
| at scala.collection.IterableLike.foreach$(IterableLike.scala:73) | |
| at scala.collection.AbstractIterable.foreach(Iterable.scala:56) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processLine(SparkSQLCLIDriver.scala:490) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:282) | |
| at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) | |
| at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:928) | |
| at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) | |
| at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) | |
| at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) | |
| at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1007) | |
| at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1016) | |
| at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) | |
| Caused by: java.lang.IllegalArgumentException: Compression codec com.hadoop.compression.lzo.LzoCodec not found. | |
| at org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses(CompressionCodecFactory.java:139) | |
| at org.apache.hadoop.io.compress.CompressionCodecFactory.<init>(CompressionCodecFactory.java:180) | |
| at org.apache.spark.sql.execution.datasources.TextBasedFileFormat.isSplitable(FileFormat.scala:179) | |
| at org.apache.spark.sql.execution.datasources.json.JsonFileFormat.isSplitable(JsonFileFormat.scala:49) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.$anonfun$createNonBucketedReadRDD$3(DataSourceScanExec.scala:575) | |
| at scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:245) | |
| at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) | |
| at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) | |
| at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38) | |
| at scala.collection.TraversableLike.flatMap(TraversableLike.scala:245) | |
| at scala.collection.TraversableLike.flatMap$(TraversableLike.scala:242) | |
| at scala.collection.AbstractTraversable.flatMap(Traversable.scala:108) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.$anonfun$createNonBucketedReadRDD$2(DataSourceScanExec.scala:571) | |
| at scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:245) | |
| at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) | |
| at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) | |
| at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198) | |
| at scala.collection.TraversableLike.flatMap(TraversableLike.scala:245) | |
| at scala.collection.TraversableLike.flatMap$(TraversableLike.scala:242) | |
| at scala.collection.mutable.ArrayOps$ofRef.flatMap(ArrayOps.scala:198) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.createNonBucketedReadRDD(DataSourceScanExec.scala:570) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:405) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:390) | |
| at org.apache.spark.sql.execution.FileSourceScanExec.doExecute(DataSourceScanExec.scala:473) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.DeserializeToObjectExec.doExecute(objects.scala:96) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.MapPartitionsExec.doExecute(objects.scala:192) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDD(WholeStageCodegenExec.scala:525) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(WholeStageCodegenExec.scala:453) | |
| at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(WholeStageCodegenExec.scala:452) | |
| at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:496) | |
| at org.apache.spark.sql.execution.SerializeFromObjectExec.inputRDDs(objects.scala:117) | |
| at org.apache.spark.sql.execution.ProjectExec.inputRDDs(basicPhysicalOperators.scala:47) | |
| at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) | |
| at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD$lzycompute(ShuffleExchangeExec.scala:106) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD(ShuffleExchangeExec.scala:106) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency$lzycompute(ShuffleExchangeExec.scala:139) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency(ShuffleExchangeExec.scala:137) | |
| at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.$anonfun$doExecute$1(ShuffleExchangeExec.scala:154) | |
| at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52) | |
| ... 165 more | |
| Caused by: java.lang.ClassNotFoundException: Class com.hadoop.compression.lzo.LzoCodec not found | |
| at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2499) | |
| at org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses(CompressionCodecFactory.java:132) | |
| ... 223 more | |
| spark-sql> | |
| > | |
| > | |
| > |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment