Skip to content

Instantly share code, notes, and snippets.

@DmytroMitin

DmytroMitin/App.scala

Last active March 11, 2023 05:18
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save DmytroMitin/ba469faeca2230890845e1532b36e2a1 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save DmytroMitin/ba469faeca2230890845e1532b36e2a1 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
App.scala
import org.apache.spark.sql.catalyst.ScalaReflection
import ScalaReflection.universe._
import org.apache.spark.sql.SparkSession
import org.reflections.Reflections
import scala.jdk.CollectionConverters._
import scala.tools.reflect.ToolBox
object App {
def getType[T: TypeTag](obj: T) = typeOf[T]
trait Base
class A extends Base {
def impl(): Function1[Int, String] = new Function1[Int, String] {
def apply(x: Int): String = "ab" + x.toString
}
val tpe = getType(impl())
}
val reflections = new Reflections()
val classes: List[Class[_ <: Base]] = reflections.getSubTypesOf(classOf[Base]).asScala.toList
val clazz = classes(0)
val instance = clazz.newInstance()
val impl = clazz.getDeclaredMethod("impl").invoke(instance)
val List(argType, returnType) = clazz.getDeclaredMethod("tpe").invoke(instance).asInstanceOf[Type].typeArgs
val spark = SparkSession.builder()
.master("local")
.appName("Spark app")
.getOrCreate()
val rm = ScalaReflection.mirror
val tb = rm.mkToolBox()
tb.eval(q"""App.spark.udf.register("foo", App.impl.asInstanceOf[$argType => $returnType])""")
def main(args: Array[String]): Unit = {
spark.sql("""SELECT foo(10)""").show()
}
}
Raw
build.sbt
ThisBuild / version := "0.1.0-SNAPSHOT"
ThisBuild / scalaVersion := "2.13.10"
lazy val root = (project in file("."))
.settings(
name := "scalademo"
)
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-sql" % "3.3.1",
scalaOrganization.value % "scala-compiler" % scalaVersion.value,
"org.reflections" % "reflections" % "0.10.2",
)
scalacOptions ++= Seq(
"-deprecation",
)
//fork := true
Raw
log.txt
/usr/local/jdk1.8.0_162/bin/java -javaagent:/media/data/idea-IU-211.7628.21/lib/idea_rt.jar=45253:/media/data/idea-IU-211.7628.21/bin -Dfile.encoding=UTF-8 -classpath /usr/local/jdk1.8.0_162/jre/lib/charsets.jar:/usr/local/jdk1.8.0_162/jre/lib/deploy.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/cldrdata.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/dnsns.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/jaccess.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/jfxrt.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/localedata.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/nashorn.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/sunec.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/sunjce_provider.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/sunpkcs11.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/zipfs.jar:/usr/local/jdk1.8.0_162/jre/lib/javaws.jar:/usr/local/jdk1.8.0_162/jre/lib/jce.jar:/usr/local/jdk1.8.0_162/jre/lib/jfr.jar:/usr/local/jdk1.8.0_162/jre/lib/jfxswt.jar:/usr/local/jdk1.8.0_162/jre/lib/jsse.jar:/usr/local/jdk1.8.0_162/jre/lib/management-agent.jar:/usr/local/jdk1.8.0_162/jre/lib/plugin.jar:/usr/local/jdk1.8.0_162/jre/lib/resources.jar:/usr/local/jdk1.8.0_162/jre/lib/rt.jar:/media/data/Projects1/scalademo35/target/scala-2.13/classes:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/clearspring/analytics/stream/2.9.6/stream-2.9.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/esotericsoftware/kryo-shaded/4.0.2/kryo-shaded-4.0.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/esotericsoftware/minlog/1.3.0/minlog-1.3.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/2.13.4/jackson-annotations-2.13.4.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.13.4/jackson-core-2.13.4.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/2.13.4.1/jackson-databind-2.13.4.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-scala_2.13/2.13.4/jackson-module-scala_2.13-2.13.4.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/github/luben/zstd-jni/1.5.2-1/zstd-jni-1.5.2-1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/code/gson/gson/2.8.6/gson-2.8.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/crypto/tink/tink/1.6.1/tink-1.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/flatbuffers/flatbuffers-java/1.12.0/flatbuffers-java-1.12.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/guava/guava/16.0.1/guava-16.0.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.14.0/protobuf-java-3.14.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/ning/compress-lzf/1.1/compress-lzf-1.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/thoughtworks/paranamer/paranamer/2.8/paranamer-2.8.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/twitter/chill-java/0.10.0/chill-java-0.10.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/twitter/chill_2.13/0.10.0/chill_2.13-0.10.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/univocity/univocity-parsers/2.9.1/univocity-parsers-2.9.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/commons-io/commons-io/2.11.0/commons-io-2.11.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/airlift/aircompressor/0.21/aircompressor-0.21.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/dropwizard/metrics/metrics-core/4.2.7/metrics-core-4.2.7.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/dropwizard/metrics/metrics-graphite/4.2.7/metrics-graphite-4.2.7.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/dropwizard/metrics/metrics-jmx/4.2.7/metrics-jmx-4.2.7.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/dropwizard/metrics/metrics-json/4.2.7/metrics-json-4.2.7.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/dropwizard/metrics/metrics-jvm/4.2.7/metrics-jvm-4.2.7.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/github/java-diff-utils/java-diff-utils/4.12/java-diff-utils-4.12.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-all/4.1.74.Final/netty-all-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-buffer/4.1.74.Final/netty-buffer-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-codec/4.1.74.Final/netty-codec-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-common/4.1.74.Final/netty-common-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-handler/4.1.74.Final/netty-handler-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-resolver/4.1.74.Final/netty-resolver-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-tcnative-classes/2.0.48.Final/netty-tcnative-classes-2.0.48.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-classes-epoll/4.1.74.Final/netty-transport-classes-epoll-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-classes-kqueue/4.1.74.Final/netty-transport-classes-kqueue-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-epoll/4.1.74.Final/netty-transport-native-epoll-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-epoll/4.1.74.Final/netty-transport-native-epoll-4.1.74.Final-linux-aarch_64.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-epoll/4.1.74.Final/netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-kqueue/4.1.74.Final/netty-transport-native-kqueue-4.1.74.Final-osx-aarch_64.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-kqueue/4.1.74.Final/netty-transport-native-kqueue-4.1.74.Final-osx-x86_64.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-unix-common/4.1.74.Final/netty-transport-native-unix-common-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport/4.1.74.Final/netty-transport-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/jakarta/annotation/jakarta.annotation-api/1.3.5/jakarta.annotation-api-1.3.5.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/jakarta/servlet/jakarta.servlet-api/4.0.3/jakarta.servlet-api-4.0.3.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/jakarta/validation/jakarta.validation-api/2.0.2/jakarta.validation-api-2.0.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/jakarta/ws/rs/jakarta.ws.rs-api/2.1.6/jakarta.ws.rs-api-2.1.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/javax/activation/activation/1.1.1/activation-1.1.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/net/java/dev/jna/jna/5.9.0/jna-5.9.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/net/razorvine/pickle/1.2/pickle-1.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/net/sf/py4j/py4j/0.10.9.5/py4j-0.10.9.5.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/antlr/antlr4-runtime/4.8/antlr4-runtime-4.8.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/arrow/arrow-format/7.0.0/arrow-format-7.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/arrow/arrow-memory-core/7.0.0/arrow-memory-core-7.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/arrow/arrow-memory-netty/7.0.0/arrow-memory-netty-7.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/arrow/arrow-vector/7.0.0/arrow-vector-7.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/avro/avro-ipc/1.11.0/avro-ipc-1.11.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/avro/avro-mapred/1.11.0/avro-mapred-1.11.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/avro/avro/1.11.0/avro-1.11.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-collections4/4.4/commons-collections4-4.4.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-crypto/1.1.0/commons-crypto-1.1.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-math3/3.6.1/commons-math3-3.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-text/1.9/commons-text-1.9.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-api/3.3.2/hadoop-client-api-3.3.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-runtime/3.3.2/hadoop-client-runtime-3.3.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/hive/hive-storage-api/2.7.2/hive-storage-api-2.7.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/ivy/ivy/2.5.0/ivy-2.5.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-1.2-api/2.17.2/log4j-1.2-api-2.17.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.17.2/log4j-api-2.17.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.17.2/log4j-core-2.17.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-slf4j-impl/2.17.2/log4j-slf4j-impl-2.17.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/orc/orc-core/1.7.6/orc-core-1.7.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/orc/orc-mapreduce/1.7.6/orc-mapreduce-1.7.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/orc/orc-shims/1.7.6/orc-shims-1.7.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-column/1.12.2/parquet-column-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-common/1.12.2/parquet-common-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-encoding/1.12.2/parquet-encoding-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-format-structures/1.12.2/parquet-format-structures-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-hadoop/1.12.2/parquet-hadoop-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-jackson/1.12.2/parquet-jackson-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-catalyst_2.13/3.3.1/spark-catalyst_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-core_2.13/3.3.1/spark-core_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-kvstore_2.13/3.3.1/spark-kvstore_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-launcher_2.13/3.3.1/spark-launcher_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-network-common_2.13/3.3.1/spark-network-common_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-network-shuffle_2.13/3.3.1/spark-network-shuffle_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-sketch_2.13/3.3.1/spark-sketch_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-sql_2.13/3.3.1/spark-sql_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-tags_2.13/3.3.1/spark-tags_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-unsafe_2.13/3.3.1/spark-unsafe_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/xbean/xbean-asm9-shaded/4.20/xbean-asm9-shaded-4.20.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/yetus/audience-annotations/0.12.0/audience-annotations-0.12.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/zookeeper/zookeeper-jute/3.6.2/zookeeper-jute-3.6.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/zookeeper/zookeeper/3.6.2/zookeeper-3.6.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/codehaus/janino/commons-compiler/3.0.16/commons-compiler-3.0.16.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/codehaus/janino/janino/3.0.16/janino-3.0.16.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged/2.6.1/aopalliance-repackaged-2.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject/2.6.1/jakarta.inject-2.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/hk2-api/2.6.1/hk2-api-2.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator/2.6.1/hk2-locator-2.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils/2.6.1/hk2-utils-2.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator/1.0.3/osgi-resource-locator-1.0.3.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core/2.36/jersey-container-servlet-core-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet/2.36/jersey-container-servlet-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client/2.36/jersey-client-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common/2.36/jersey-common-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server/2.36/jersey-server-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2/2.36/jersey-hk2-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/javassist/javassist/3.28.0-GA/javassist-3.28.0-GA.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jetbrains/annotations/17.0.0/annotations-17.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jline/jline/3.21.0/jline-3.21.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/json4s/json4s-ast_2.13/3.7.0-M11/json4s-ast_2.13-3.7.0-M11.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/json4s/json4s-core_2.13/3.7.0-M11/json4s-core_2.13-3.7.0-M11.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/json4s/json4s-jackson_2.13/3.7.0-M11/json4s-jackson_2.13-3.7.0-M11.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/json4s/json4s-scalap_2.13/3.7.0-M11/json4s-scalap_2.13-3.7.0-M11.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/lz4/lz4-java/1.8.0/lz4-java-1.8.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/objenesis/objenesis/2.5.1/objenesis-2.5.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/reflections/reflections/0.10.2/reflections-0.10.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/roaringbitmap/RoaringBitmap/0.9.25/RoaringBitmap-0.9.25.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/roaringbitmap/shims/0.9.25/shims-0.9.25.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/rocksdb/rocksdbjni/6.20.3/rocksdbjni-6.20.3.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-parallel-collections_2.13/1.0.3/scala-parallel-collections_2.13-1.0.3.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-parser-combinators_2.13/1.1.2/scala-parser-combinators_2.13-1.1.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-xml_2.13/1.2.0/scala-xml_2.13-1.2.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.10/scala-compiler-2.13.10.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-reflect/2.13.10/scala-reflect-2.13.10.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/slf4j/jcl-over-slf4j/1.7.32/jcl-over-slf4j-1.7.32.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/slf4j/jul-to-slf4j/1.7.32/jul-to-slf4j-1.7.32.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/slf4j/slf4j-api/1.7.35/slf4j-api-1.7.35.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/spark-project/spark/unused/1.0.0/unused-1.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/threeten/threeten-extra/1.5.0/threeten-extra-1.5.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/tukaani/xz/1.9/xz-1.9.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/xerial/snappy/snappy-java/1.1.8.4/snappy-java-1.1.8.4.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/oro/oro/2.0.8/oro-2.0.8.jar App23
Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties
23/03/11 07:15:54 WARN Utils: Your hostname, dmitin-HP-Pavilion-Laptop resolves to a loopback address: 127.0.1.1; using 192.168.0.105 instead (on interface wlo1)
23/03/11 07:15:54 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address
23/03/11 07:15:54 INFO SparkContext: Running Spark version 3.3.1
23/03/11 07:15:54 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
23/03/11 07:15:54 INFO ResourceUtils: ==============================================================
23/03/11 07:15:54 INFO ResourceUtils: No custom resources configured for spark.driver.
23/03/11 07:15:54 INFO ResourceUtils: ==============================================================
23/03/11 07:15:54 INFO SparkContext: Submitted application: Spark app
23/03/11 07:15:54 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0)
23/03/11 07:15:54 INFO ResourceProfile: Limiting resource is cpu
23/03/11 07:15:54 INFO ResourceProfileManager: Added ResourceProfile id: 0
23/03/11 07:15:54 INFO SecurityManager: Changing view acls to: dmitin
23/03/11 07:15:54 INFO SecurityManager: Changing modify acls to: dmitin
23/03/11 07:15:54 INFO SecurityManager: Changing view acls groups to:
23/03/11 07:15:54 INFO SecurityManager: Changing modify acls groups to:
23/03/11 07:15:54 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(dmitin); groups with view permissions: Set(); users with modify permissions: Set(dmitin); groups with modify permissions: Set()
23/03/11 07:15:55 INFO Utils: Successfully started service 'sparkDriver' on port 35331.
23/03/11 07:15:55 INFO SparkEnv: Registering MapOutputTracker
23/03/11 07:15:55 INFO SparkEnv: Registering BlockManagerMaster
23/03/11 07:15:55 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
23/03/11 07:15:55 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up
23/03/11 07:15:55 INFO SparkEnv: Registering BlockManagerMasterHeartbeat
23/03/11 07:15:55 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-4d6c7237-02ee-4d46-9a47-a29ef52f707e
23/03/11 07:15:55 INFO MemoryStore: MemoryStore started with capacity 1944.0 MiB
23/03/11 07:15:55 INFO SparkEnv: Registering OutputCommitCoordinator
23/03/11 07:15:55 INFO Utils: Successfully started service 'SparkUI' on port 4040.
23/03/11 07:15:55 INFO Executor: Starting executor ID driver on host 192.168.0.105
23/03/11 07:15:55 INFO Executor: Starting executor with user classpath (userClassPathFirst = false): ''
23/03/11 07:15:55 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 40451.
23/03/11 07:15:55 INFO NettyBlockTransferService: Server created on 192.168.0.105:40451
23/03/11 07:15:55 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
23/03/11 07:15:55 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.0.105, 40451, None)
23/03/11 07:15:55 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.0.105:40451 with 1944.0 MiB RAM, BlockManagerId(driver, 192.168.0.105, 40451, None)
23/03/11 07:15:55 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.0.105, 40451, None)
23/03/11 07:15:55 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.0.105, 40451, None)
23/03/11 07:15:59 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir.
23/03/11 07:15:59 INFO SharedState: Warehouse path is 'file:/media/data/Projects1/scalademo35/spark-warehouse'.
23/03/11 07:16:01 INFO CodeGenerator: Code generated in 233.339743 ms
Exception in thread "main" org.apache.spark.SparkException: Task not serializable
at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:444)
at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:416)
at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:163)
at org.apache.spark.SparkContext.clean(SparkContext.scala:2491)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsWithIndex$1(RDD.scala:904)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:406)
at org.apache.spark.rdd.RDD.mapPartitionsWithIndex(RDD.scala:903)
at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:754)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:194)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:232)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:229)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:190)
at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:340)
at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:421)
at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3868)
at org.apache.spark.sql.Dataset.$anonfun$head$1(Dataset.scala:2863)
at org.apache.spark.sql.Dataset.$anonfun$withAction$2(Dataset.scala:3858)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:510)
at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3856)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:109)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:169)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:95)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3856)
at org.apache.spark.sql.Dataset.head(Dataset.scala:2863)
at org.apache.spark.sql.Dataset.take(Dataset.scala:3084)
at org.apache.spark.sql.Dataset.getRows(Dataset.scala:288)
at org.apache.spark.sql.Dataset.showString(Dataset.scala:327)
at org.apache.spark.sql.Dataset.show(Dataset.scala:808)
at org.apache.spark.sql.Dataset.show(Dataset.scala:767)
at org.apache.spark.sql.Dataset.show(Dataset.scala:776)
at App23$.main(App23.scala:38)
at App23.main(App23.scala)
Caused by: java.io.NotSerializableException: App23$A$$anon$1
Serialization stack:
- object not serializable (class: App23$A$$anon$1, value: <function1>)
- element of array (index: 1)
- array (class [Ljava.lang.Object;, size 4)
- field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
- object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.spark.sql.catalyst.expressions.ScalaUDF, functionalInterfaceMethod=scala/Function1.apply:(Ljava/lang/Object;)Ljava/lang/Object;, implementation=invokeStatic org/apache/spark/sql/catalyst/expressions/ScalaUDF.$anonfun$f$2:(Lorg/apache/spark/sql/catalyst/expressions/ScalaUDF;Lscala/Function1;Lorg/apache/spark/sql/catalyst/expressions/Expression;Lscala/runtime/LazyRef;Lorg/apache/spark/sql/catalyst/InternalRow;)Ljava/lang/Object;, instantiatedMethodType=(Lorg/apache/spark/sql/catalyst/InternalRow;)Ljava/lang/Object;, numCaptured=4])
- writeReplace data (class: java.lang.invoke.SerializedLambda)
- object (class org.apache.spark.sql.catalyst.expressions.ScalaUDF$$Lambda$1781/1548823953, org.apache.spark.sql.catalyst.expressions.ScalaUDF$$Lambda$1781/1548823953@6793a49c)
- field (class: org.apache.spark.sql.catalyst.expressions.ScalaUDF, name: f, type: interface scala.Function1)
- object (class org.apache.spark.sql.catalyst.expressions.ScalaUDF, foo(10))
- field (class: org.apache.spark.sql.catalyst.expressions.ScalaUDF, name: canonicalized, type: class org.apache.spark.sql.catalyst.expressions.Expression)
- object (class org.apache.spark.sql.catalyst.expressions.ScalaUDF, foo(10))
- element of array (index: 0)
- array (class [Ljava.lang.Object;, size 2)
- field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
- object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.spark.sql.catalyst.expressions.ScalaUDF, functionalInterfaceMethod=scala/Function1.apply:(Ljava/lang/Object;)Ljava/lang/Object;, implementation=invokeStatic org/apache/spark/sql/catalyst/expressions/ScalaUDF.$anonfun$catalystConverter$3:(Lorg/apache/spark/sql/catalyst/expressions/ScalaUDF;Lscala/Function1;Ljava/lang/Object;)Ljava/lang/Object;, instantiatedMethodType=(Ljava/lang/Object;)Ljava/lang/Object;, numCaptured=2])
- writeReplace data (class: java.lang.invoke.SerializedLambda)
- object (class org.apache.spark.sql.catalyst.expressions.ScalaUDF$$Lambda$1784/282777364, org.apache.spark.sql.catalyst.expressions.ScalaUDF$$Lambda$1784/282777364@6537ab7d)
- element of array (index: 1)
- array (class [Lscala.Function1;, size 2)
- element of array (index: 1)
- array (class [Ljava.lang.Object;, size 3)
- element of array (index: 1)
- array (class [Ljava.lang.Object;, size 3)
- field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
- object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.spark.sql.execution.WholeStageCodegenExec, functionalInterfaceMethod=scala/Function2.apply:(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;, implementation=invokeStatic org/apache/spark/sql/execution/WholeStageCodegenExec.$anonfun$doExecute$4$adapted:(Lorg/apache/spark/sql/catalyst/expressions/codegen/CodeAndComment;[Ljava/lang/Object;Lorg/apache/spark/sql/execution/metric/SQLMetric;Ljava/lang/Object;Lscala/collection/Iterator;)Lscala/collection/Iterator;, instantiatedMethodType=(Ljava/lang/Object;Lscala/collection/Iterator;)Lscala/collection/Iterator;, numCaptured=3])
- writeReplace data (class: java.lang.invoke.SerializedLambda)
- object (class org.apache.spark.sql.execution.WholeStageCodegenExec$$Lambda$2458/112026756, org.apache.spark.sql.execution.WholeStageCodegenExec$$Lambda$2458/112026756@57b03187)
at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:41)
at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:49)
at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:115)
at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:441)
... 36 more
23/03/11 07:16:02 INFO SparkContext: Invoking stop() from shutdown hook
23/03/11 07:16:02 INFO SparkUI: Stopped Spark web UI at http://192.168.0.105:4040
23/03/11 07:16:02 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
23/03/11 07:16:02 INFO MemoryStore: MemoryStore cleared
23/03/11 07:16:02 INFO BlockManager: BlockManager stopped
23/03/11 07:16:02 INFO BlockManagerMaster: BlockManagerMaster stopped
23/03/11 07:16:02 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
23/03/11 07:16:02 INFO SparkContext: Successfully stopped SparkContext
23/03/11 07:16:02 INFO ShutdownHookManager: Shutdown hook called
23/03/11 07:16:02 INFO ShutdownHookManager: Deleting directory /tmp/spark-766a6ccc-64a1-480e-9eea-61285748f713
Process finished with exit code 1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment