Last active
March 11, 2023 04:12
-
-
Save DmytroMitin/5b5dd4d7db0d0eebb51dd8c16735e0fb to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import org.apache.spark.sql.SparkSession | |
| import org.apache.spark.sql.catalyst.ScalaReflection | |
| import ScalaReflection.universe._ | |
| import scala.tools.reflect.ToolBox | |
| object App { | |
| val spark = SparkSession.builder | |
| .master("local") | |
| .appName("Spark app") | |
| .getOrCreate() | |
| def impl(): Any = (x: Int) => "ab" + x.toString | |
| val ttag1 = typeTag[Int] | |
| val ttag = typeTag[String] | |
| val rm = ScalaReflection.mirror | |
| val tb = rm.mkToolBox() | |
| tb.eval(q"""App.spark.udf.register("foo", App.impl().asInstanceOf[$ttag1 => $ttag])""") | |
| def main(args: Array[String]): Unit = { | |
| spark.sql("""SELECT foo(10)""").show() | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ThisBuild / version := "0.1.0-SNAPSHOT" | |
| ThisBuild / scalaVersion := "2.13.10" | |
| lazy val root = (project in file(".")) | |
| .settings( | |
| name := "scalademo" | |
| ) | |
| libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.3.1" | |
| libraryDependencies += scalaOrganization.value % "scala-compiler" % scalaVersion.value |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| /usr/local/jdk1.8.0_162/bin/java -javaagent:/media/data/idea-IU-211.7628.21/lib/idea_rt.jar=38309:/media/data/idea-IU-211.7628.21/bin -Dfile.encoding=UTF-8 -classpath /usr/local/jdk1.8.0_162/jre/lib/charsets.jar:/usr/local/jdk1.8.0_162/jre/lib/deploy.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/cldrdata.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/dnsns.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/jaccess.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/jfxrt.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/localedata.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/nashorn.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/sunec.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/sunjce_provider.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/sunpkcs11.jar:/usr/local/jdk1.8.0_162/jre/lib/ext/zipfs.jar:/usr/local/jdk1.8.0_162/jre/lib/javaws.jar:/usr/local/jdk1.8.0_162/jre/lib/jce.jar:/usr/local/jdk1.8.0_162/jre/lib/jfr.jar:/usr/local/jdk1.8.0_162/jre/lib/jfxswt.jar:/usr/local/jdk1.8.0_162/jre/lib/jsse.jar:/usr/local/jdk1.8.0_162/jre/lib/management-agent.jar:/usr/local/jdk1.8.0_162/jre/lib/plugin.jar:/usr/local/jdk1.8.0_162/jre/lib/resources.jar:/usr/local/jdk1.8.0_162/jre/lib/rt.jar:/media/data/Projects1/scalademo35/target/scala-2.13/classes:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/clearspring/analytics/stream/2.9.6/stream-2.9.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/esotericsoftware/kryo-shaded/4.0.2/kryo-shaded-4.0.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/esotericsoftware/minlog/1.3.0/minlog-1.3.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/2.13.4/jackson-annotations-2.13.4.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.13.4/jackson-core-2.13.4.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/2.13.4.1/jackson-databind-2.13.4.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-scala_2.13/2.13.4/jackson-module-scala_2.13-2.13.4.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/github/luben/zstd-jni/1.5.2-1/zstd-jni-1.5.2-1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/code/gson/gson/2.8.6/gson-2.8.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/crypto/tink/tink/1.6.1/tink-1.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/flatbuffers/flatbuffers-java/1.12.0/flatbuffers-java-1.12.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/guava/guava/16.0.1/guava-16.0.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.14.0/protobuf-java-3.14.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/ning/compress-lzf/1.1/compress-lzf-1.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/thoughtworks/paranamer/paranamer/2.8/paranamer-2.8.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/twitter/chill-java/0.10.0/chill-java-0.10.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/twitter/chill_2.13/0.10.0/chill_2.13-0.10.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/com/univocity/univocity-parsers/2.9.1/univocity-parsers-2.9.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/commons-io/commons-io/2.11.0/commons-io-2.11.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/airlift/aircompressor/0.21/aircompressor-0.21.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/dropwizard/metrics/metrics-core/4.2.7/metrics-core-4.2.7.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/dropwizard/metrics/metrics-graphite/4.2.7/metrics-graphite-4.2.7.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/dropwizard/metrics/metrics-jmx/4.2.7/metrics-jmx-4.2.7.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/dropwizard/metrics/metrics-json/4.2.7/metrics-json-4.2.7.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/dropwizard/metrics/metrics-jvm/4.2.7/metrics-jvm-4.2.7.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/github/java-diff-utils/java-diff-utils/4.12/java-diff-utils-4.12.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-all/4.1.74.Final/netty-all-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-buffer/4.1.74.Final/netty-buffer-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-codec/4.1.74.Final/netty-codec-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-common/4.1.74.Final/netty-common-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-handler/4.1.74.Final/netty-handler-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-resolver/4.1.74.Final/netty-resolver-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-tcnative-classes/2.0.48.Final/netty-tcnative-classes-2.0.48.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-classes-epoll/4.1.74.Final/netty-transport-classes-epoll-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-classes-kqueue/4.1.74.Final/netty-transport-classes-kqueue-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-epoll/4.1.74.Final/netty-transport-native-epoll-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-epoll/4.1.74.Final/netty-transport-native-epoll-4.1.74.Final-linux-aarch_64.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-epoll/4.1.74.Final/netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-kqueue/4.1.74.Final/netty-transport-native-kqueue-4.1.74.Final-osx-aarch_64.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-kqueue/4.1.74.Final/netty-transport-native-kqueue-4.1.74.Final-osx-x86_64.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport-native-unix-common/4.1.74.Final/netty-transport-native-unix-common-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/io/netty/netty-transport/4.1.74.Final/netty-transport-4.1.74.Final.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/jakarta/annotation/jakarta.annotation-api/1.3.5/jakarta.annotation-api-1.3.5.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/jakarta/servlet/jakarta.servlet-api/4.0.3/jakarta.servlet-api-4.0.3.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/jakarta/validation/jakarta.validation-api/2.0.2/jakarta.validation-api-2.0.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/jakarta/ws/rs/jakarta.ws.rs-api/2.1.6/jakarta.ws.rs-api-2.1.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/javax/activation/activation/1.1.1/activation-1.1.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/net/java/dev/jna/jna/5.9.0/jna-5.9.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/net/razorvine/pickle/1.2/pickle-1.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/net/sf/py4j/py4j/0.10.9.5/py4j-0.10.9.5.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/antlr/antlr4-runtime/4.8/antlr4-runtime-4.8.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/arrow/arrow-format/7.0.0/arrow-format-7.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/arrow/arrow-memory-core/7.0.0/arrow-memory-core-7.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/arrow/arrow-memory-netty/7.0.0/arrow-memory-netty-7.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/arrow/arrow-vector/7.0.0/arrow-vector-7.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/avro/avro-ipc/1.11.0/avro-ipc-1.11.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/avro/avro-mapred/1.11.0/avro-mapred-1.11.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/avro/avro/1.11.0/avro-1.11.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-collections4/4.4/commons-collections4-4.4.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-crypto/1.1.0/commons-crypto-1.1.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-math3/3.6.1/commons-math3-3.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-text/1.9/commons-text-1.9.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-api/3.3.2/hadoop-client-api-3.3.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-runtime/3.3.2/hadoop-client-runtime-3.3.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/hive/hive-storage-api/2.7.2/hive-storage-api-2.7.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/ivy/ivy/2.5.0/ivy-2.5.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-1.2-api/2.17.2/log4j-1.2-api-2.17.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.17.2/log4j-api-2.17.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.17.2/log4j-core-2.17.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-slf4j-impl/2.17.2/log4j-slf4j-impl-2.17.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/orc/orc-core/1.7.6/orc-core-1.7.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/orc/orc-mapreduce/1.7.6/orc-mapreduce-1.7.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/orc/orc-shims/1.7.6/orc-shims-1.7.6.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-column/1.12.2/parquet-column-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-common/1.12.2/parquet-common-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-encoding/1.12.2/parquet-encoding-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-format-structures/1.12.2/parquet-format-structures-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-hadoop/1.12.2/parquet-hadoop-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/parquet/parquet-jackson/1.12.2/parquet-jackson-1.12.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-catalyst_2.13/3.3.1/spark-catalyst_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-core_2.13/3.3.1/spark-core_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-kvstore_2.13/3.3.1/spark-kvstore_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-launcher_2.13/3.3.1/spark-launcher_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-network-common_2.13/3.3.1/spark-network-common_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-network-shuffle_2.13/3.3.1/spark-network-shuffle_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-sketch_2.13/3.3.1/spark-sketch_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-sql_2.13/3.3.1/spark-sql_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-tags_2.13/3.3.1/spark-tags_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/spark/spark-unsafe_2.13/3.3.1/spark-unsafe_2.13-3.3.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/xbean/xbean-asm9-shaded/4.20/xbean-asm9-shaded-4.20.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/yetus/audience-annotations/0.12.0/audience-annotations-0.12.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/zookeeper/zookeeper-jute/3.6.2/zookeeper-jute-3.6.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/apache/zookeeper/zookeeper/3.6.2/zookeeper-3.6.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/codehaus/janino/commons-compiler/3.0.16/commons-compiler-3.0.16.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/codehaus/janino/janino/3.0.16/janino-3.0.16.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged/2.6.1/aopalliance-repackaged-2.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject/2.6.1/jakarta.inject-2.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/hk2-api/2.6.1/hk2-api-2.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator/2.6.1/hk2-locator-2.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils/2.6.1/hk2-utils-2.6.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator/1.0.3/osgi-resource-locator-1.0.3.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core/2.36/jersey-container-servlet-core-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet/2.36/jersey-container-servlet-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client/2.36/jersey-client-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common/2.36/jersey-common-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server/2.36/jersey-server-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/glassfish/jersey/inject/jersey-hk2/2.36/jersey-hk2-2.36.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/javassist/javassist/3.25.0-GA/javassist-3.25.0-GA.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jetbrains/annotations/17.0.0/annotations-17.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jline/jline/3.21.0/jline-3.21.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/json4s/json4s-ast_2.13/3.7.0-M11/json4s-ast_2.13-3.7.0-M11.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/json4s/json4s-core_2.13/3.7.0-M11/json4s-core_2.13-3.7.0-M11.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/json4s/json4s-jackson_2.13/3.7.0-M11/json4s-jackson_2.13-3.7.0-M11.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/json4s/json4s-scalap_2.13/3.7.0-M11/json4s-scalap_2.13-3.7.0-M11.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/lz4/lz4-java/1.8.0/lz4-java-1.8.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/objenesis/objenesis/2.5.1/objenesis-2.5.1.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/roaringbitmap/RoaringBitmap/0.9.25/RoaringBitmap-0.9.25.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/roaringbitmap/shims/0.9.25/shims-0.9.25.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/rocksdb/rocksdbjni/6.20.3/rocksdbjni-6.20.3.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-parallel-collections_2.13/1.0.3/scala-parallel-collections_2.13-1.0.3.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-parser-combinators_2.13/1.1.2/scala-parser-combinators_2.13-1.1.2.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-xml_2.13/1.2.0/scala-xml_2.13-1.2.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.10/scala-compiler-2.13.10.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-reflect/2.13.10/scala-reflect-2.13.10.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/slf4j/jcl-over-slf4j/1.7.32/jcl-over-slf4j-1.7.32.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/slf4j/jul-to-slf4j/1.7.32/jul-to-slf4j-1.7.32.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/slf4j/slf4j-api/1.7.35/slf4j-api-1.7.35.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/spark-project/spark/unused/1.0.0/unused-1.0.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/threeten/threeten-extra/1.5.0/threeten-extra-1.5.0.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/tukaani/xz/1.9/xz-1.9.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/org/xerial/snappy/snappy-java/1.1.8.4/snappy-java-1.1.8.4.jar:/home/dmitin/.cache/coursier/v1/https/repo1.maven.org/maven2/oro/oro/2.0.8/oro-2.0.8.jar App22 | |
| Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties | |
| 23/03/11 06:08:58 WARN Utils: Your hostname, dmitin-HP-Pavilion-Laptop resolves to a loopback address: 127.0.1.1; using 192.168.0.105 instead (on interface wlo1) | |
| 23/03/11 06:08:58 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address | |
| 23/03/11 06:08:58 INFO SparkContext: Running Spark version 3.3.1 | |
| 23/03/11 06:08:58 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable | |
| 23/03/11 06:08:58 INFO ResourceUtils: ============================================================== | |
| 23/03/11 06:08:58 INFO ResourceUtils: No custom resources configured for spark.driver. | |
| 23/03/11 06:08:58 INFO ResourceUtils: ============================================================== | |
| 23/03/11 06:08:58 INFO SparkContext: Submitted application: Spark app | |
| 23/03/11 06:08:58 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0) | |
| 23/03/11 06:08:58 INFO ResourceProfile: Limiting resource is cpu | |
| 23/03/11 06:08:58 INFO ResourceProfileManager: Added ResourceProfile id: 0 | |
| 23/03/11 06:08:58 INFO SecurityManager: Changing view acls to: dmitin | |
| 23/03/11 06:08:58 INFO SecurityManager: Changing modify acls to: dmitin | |
| 23/03/11 06:08:58 INFO SecurityManager: Changing view acls groups to: | |
| 23/03/11 06:08:58 INFO SecurityManager: Changing modify acls groups to: | |
| 23/03/11 06:08:58 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(dmitin); groups with view permissions: Set(); users with modify permissions: Set(dmitin); groups with modify permissions: Set() | |
| 23/03/11 06:08:59 INFO Utils: Successfully started service 'sparkDriver' on port 38975. | |
| 23/03/11 06:08:59 INFO SparkEnv: Registering MapOutputTracker | |
| 23/03/11 06:08:59 INFO SparkEnv: Registering BlockManagerMaster | |
| 23/03/11 06:08:59 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information | |
| 23/03/11 06:08:59 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up | |
| 23/03/11 06:08:59 INFO SparkEnv: Registering BlockManagerMasterHeartbeat | |
| 23/03/11 06:08:59 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-ea719e63-696f-4fb3-b1d1-974768a3d80a | |
| 23/03/11 06:08:59 INFO MemoryStore: MemoryStore started with capacity 1944.0 MiB | |
| 23/03/11 06:08:59 INFO SparkEnv: Registering OutputCommitCoordinator | |
| 23/03/11 06:08:59 INFO Utils: Successfully started service 'SparkUI' on port 4040. | |
| 23/03/11 06:08:59 INFO Executor: Starting executor ID driver on host 192.168.0.105 | |
| 23/03/11 06:08:59 INFO Executor: Starting executor with user classpath (userClassPathFirst = false): '' | |
| 23/03/11 06:08:59 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 32813. | |
| 23/03/11 06:08:59 INFO NettyBlockTransferService: Server created on 192.168.0.105:32813 | |
| 23/03/11 06:08:59 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy | |
| 23/03/11 06:08:59 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.0.105, 32813, None) | |
| 23/03/11 06:08:59 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.0.105:32813 with 1944.0 MiB RAM, BlockManagerId(driver, 192.168.0.105, 32813, None) | |
| 23/03/11 06:08:59 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.0.105, 32813, None) | |
| 23/03/11 06:08:59 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.0.105, 32813, None) | |
| 23/03/11 06:09:03 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir. | |
| 23/03/11 06:09:03 INFO SharedState: Warehouse path is 'file:/media/data/Projects1/scalademo35/spark-warehouse'. | |
| 23/03/11 06:09:06 INFO CodeGenerator: Code generated in 211.766308 ms | |
| 23/03/11 06:09:06 INFO SparkContext: Starting job: show at App22.scala:21 | |
| 23/03/11 06:09:06 INFO DAGScheduler: Got job 0 (show at App22.scala:21) with 1 output partitions | |
| 23/03/11 06:09:06 INFO DAGScheduler: Final stage: ResultStage 0 (show at App22.scala:21) | |
| 23/03/11 06:09:06 INFO DAGScheduler: Parents of final stage: List() | |
| 23/03/11 06:09:06 INFO DAGScheduler: Missing parents: List() | |
| 23/03/11 06:09:06 INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[2] at show at App22.scala:21), which has no missing parents | |
| 23/03/11 06:09:06 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 16.0 KiB, free 1944.0 MiB) | |
| 23/03/11 06:09:06 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 7.7 KiB, free 1944.0 MiB) | |
| 23/03/11 06:09:06 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.0.105:32813 (size: 7.7 KiB, free: 1944.0 MiB) | |
| 23/03/11 06:09:06 INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1513 | |
| 23/03/11 06:09:06 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at show at App22.scala:21) (first 15 tasks are for partitions Vector(0)) | |
| 23/03/11 06:09:06 INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks resource profile 0 | |
| 23/03/11 06:09:07 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0) (192.168.0.105, executor driver, partition 0, PROCESS_LOCAL, 7671 bytes) taskResourceAssignments Map() | |
| 23/03/11 06:09:07 INFO Executor: Running task 0.0 in stage 0.0 (TID 0) | |
| 23/03/11 06:09:07 INFO CodeGenerator: Code generated in 15.788872 ms | |
| 23/03/11 06:09:07 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 1552 bytes result sent to driver | |
| 23/03/11 06:09:07 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 415 ms on 192.168.0.105 (executor driver) (1/1) | |
| 23/03/11 06:09:07 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool | |
| 23/03/11 06:09:07 INFO DAGScheduler: ResultStage 0 (show at App22.scala:21) finished in 0,570 s | |
| 23/03/11 06:09:07 INFO DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job | |
| 23/03/11 06:09:07 INFO TaskSchedulerImpl: Killing all running tasks in stage 0: Stage finished | |
| 23/03/11 06:09:07 INFO DAGScheduler: Job 0 finished: show at App22.scala:21, took 0,615237 s | |
| 23/03/11 06:09:07 INFO CodeGenerator: Code generated in 16.727833 ms | |
| +-------+ | |
| |foo(10)| | |
| +-------+ | |
| | ab10| | |
| +-------+ | |
| 23/03/11 06:09:07 INFO SparkContext: Invoking stop() from shutdown hook | |
| 23/03/11 06:09:07 INFO SparkUI: Stopped Spark web UI at http://192.168.0.105:4040 | |
| 23/03/11 06:09:07 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! | |
| 23/03/11 06:09:07 INFO MemoryStore: MemoryStore cleared | |
| 23/03/11 06:09:07 INFO BlockManager: BlockManager stopped | |
| 23/03/11 06:09:07 INFO BlockManagerMaster: BlockManagerMaster stopped | |
| 23/03/11 06:09:07 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! | |
| 23/03/11 06:09:07 INFO SparkContext: Successfully stopped SparkContext | |
| 23/03/11 06:09:07 INFO ShutdownHookManager: Shutdown hook called | |
| 23/03/11 06:09:07 INFO ShutdownHookManager: Deleting directory /tmp/spark-0833d793-7589-4651-9ac6-f7243cb47b9f | |
| Process finished with exit code 0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment