Created
January 3, 2017 05:46
-
-
Save 1ambda/e51d2e0de570174d8441bc7c3c75f7c5 to your computer and use it in GitHub Desktop.
spark mesos packages
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| root@moby:/# spark-shell --master mesos://master.mesos:5050 --packages com.datastax.spark:spark-cassandra-connector_2.10:1.6.2,TargetHolding:pyspark-cassandra:0.3.5 --exclude-packages org.slf4j:slf4j-api | |
| Ivy Default Cache set to: /root/.ivy2/cache | |
| The jars for the packages stored in: /root/.ivy2/jars | |
| :: loading settings :: url = jar:file:/opt/spark/lib/spark-assembly-1.6.1-hadoop2.6.0.jar!/org/apache/ivy/core/settings/ivysettings.xml | |
| com.datastax.spark#spark-cassandra-connector_2.10 added as a dependency | |
| TargetHolding#pyspark-cassandra added as a dependency | |
| :: resolving dependencies :: org.apache.spark#spark-submit-parent;1.0 | |
| confs: [default] | |
| found com.datastax.spark#spark-cassandra-connector_2.10;1.6.2 in central | |
| found joda-time#joda-time;2.3 in central | |
| found com.twitter#jsr166e;1.1.0 in central | |
| found io.netty#netty-all;4.0.33.Final in central | |
| found org.joda#joda-convert;1.2 in central | |
| found org.scala-lang#scala-reflect;2.10.5 in central | |
| found TargetHolding#pyspark-cassandra;0.3.5 in spark-packages | |
| found com.datastax.spark#spark-cassandra-connector-java_2.10;1.6.0-M1 in central | |
| found org.apache.cassandra#cassandra-clientutil;3.0.2 in central | |
| found com.datastax.cassandra#cassandra-driver-core;3.0.0 in central | |
| found io.netty#netty-handler;4.0.33.Final in central | |
| found io.netty#netty-buffer;4.0.33.Final in central | |
| found io.netty#netty-common;4.0.33.Final in central | |
| found io.netty#netty-transport;4.0.33.Final in central | |
| found io.netty#netty-codec;4.0.33.Final in central | |
| found io.dropwizard.metrics#metrics-core;3.1.2 in central | |
| found org.apache.commons#commons-lang3;3.3.2 in central | |
| found com.google.guava#guava;16.0.1 in central | |
| downloading https://repo1.maven.org/maven2/com/datastax/spark/spark-cassandra-connector_2.10/1.6.2/spark-cassandra-connector_2.10-1.6.2.jar ... | |
| [SUCCESSFUL ] com.datastax.spark#spark-cassandra-connector_2.10;1.6.2!spark-cassandra-connector_2.10.jar (571ms) | |
| downloading http://dl.bintray.com/spark-packages/maven/TargetHolding/pyspark-cassandra/0.3.5/pyspark-cassandra-0.3.5.jar ... | |
| [SUCCESSFUL ] TargetHolding#pyspark-cassandra;0.3.5!pyspark-cassandra.jar (332ms) | |
| downloading https://repo1.maven.org/maven2/joda-time/joda-time/2.3/joda-time-2.3.jar ... | |
| [SUCCESSFUL ] joda-time#joda-time;2.3!joda-time.jar (66ms) | |
| downloading https://repo1.maven.org/maven2/com/twitter/jsr166e/1.1.0/jsr166e-1.1.0.jar ... | |
| [SUCCESSFUL ] com.twitter#jsr166e;1.1.0!jsr166e.jar (26ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-all/4.0.33.Final/netty-all-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-all;4.0.33.Final!netty-all.jar (157ms) | |
| downloading https://repo1.maven.org/maven2/org/joda/joda-convert/1.2/joda-convert-1.2.jar ... | |
| [SUCCESSFUL ] org.joda#joda-convert;1.2!joda-convert.jar (25ms) | |
| downloading https://repo1.maven.org/maven2/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar ... | |
| [SUCCESSFUL ] org.scala-lang#scala-reflect;2.10.5!scala-reflect.jar (236ms) | |
| downloading https://repo1.maven.org/maven2/com/datastax/spark/spark-cassandra-connector-java_2.10/1.6.0-M1/spark-cassandra-connector-java_2.10-1.6.0-M1.jar ... | |
| [SUCCESSFUL ] com.datastax.spark#spark-cassandra-connector-java_2.10;1.6.0-M1!spark-cassandra-connector-java_2.10.jar (36ms) | |
| downloading https://repo1.maven.org/maven2/org/apache/cassandra/cassandra-clientutil/3.0.2/cassandra-clientutil-3.0.2.jar ... | |
| [SUCCESSFUL ] org.apache.cassandra#cassandra-clientutil;3.0.2!cassandra-clientutil.jar (33ms) | |
| downloading https://repo1.maven.org/maven2/com/datastax/cassandra/cassandra-driver-core/3.0.0/cassandra-driver-core-3.0.0.jar ... | |
| [SUCCESSFUL ] com.datastax.cassandra#cassandra-driver-core;3.0.0!cassandra-driver-core.jar(bundle) (82ms) | |
| downloading https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.3.2/commons-lang3-3.3.2.jar ... | |
| [SUCCESSFUL ] org.apache.commons#commons-lang3;3.3.2!commons-lang3.jar (46ms) | |
| downloading https://repo1.maven.org/maven2/com/google/guava/guava/16.0.1/guava-16.0.1.jar ... | |
| [SUCCESSFUL ] com.google.guava#guava;16.0.1!guava.jar(bundle) (178ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-handler/4.0.33.Final/netty-handler-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-handler;4.0.33.Final!netty-handler.jar (49ms) | |
| downloading https://repo1.maven.org/maven2/io/dropwizard/metrics/metrics-core/3.1.2/metrics-core-3.1.2.jar ... | |
| [SUCCESSFUL ] io.dropwizard.metrics#metrics-core;3.1.2!metrics-core.jar(bundle) (38ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-buffer/4.0.33.Final/netty-buffer-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-buffer;4.0.33.Final!netty-buffer.jar (28ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-transport/4.0.33.Final/netty-transport-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-transport;4.0.33.Final!netty-transport.jar (48ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-codec/4.0.33.Final/netty-codec-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-codec;4.0.33.Final!netty-codec.jar (55ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-common/4.0.33.Final/netty-common-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-common;4.0.33.Final!netty-common.jar (43ms) | |
| :: resolution report :: resolve 4964ms :: artifacts dl 2077ms | |
| :: modules in use: | |
| TargetHolding#pyspark-cassandra;0.3.5 from spark-packages in [default] | |
| com.datastax.cassandra#cassandra-driver-core;3.0.0 from central in [default] | |
| com.datastax.spark#spark-cassandra-connector-java_2.10;1.6.0-M1 from central in [default] | |
| com.datastax.spark#spark-cassandra-connector_2.10;1.6.2 from central in [default] | |
| com.google.guava#guava;16.0.1 from central in [default] | |
| com.twitter#jsr166e;1.1.0 from central in [default] | |
| io.dropwizard.metrics#metrics-core;3.1.2 from central in [default] | |
| io.netty#netty-all;4.0.33.Final from central in [default] | |
| io.netty#netty-buffer;4.0.33.Final from central in [default] | |
| io.netty#netty-codec;4.0.33.Final from central in [default] | |
| io.netty#netty-common;4.0.33.Final from central in [default] | |
| io.netty#netty-handler;4.0.33.Final from central in [default] | |
| io.netty#netty-transport;4.0.33.Final from central in [default] | |
| joda-time#joda-time;2.3 from central in [default] | |
| org.apache.cassandra#cassandra-clientutil;3.0.2 from central in [default] | |
| org.apache.commons#commons-lang3;3.3.2 from central in [default] | |
| org.joda#joda-convert;1.2 from central in [default] | |
| org.scala-lang#scala-reflect;2.10.5 from central in [default] | |
| :: evicted modules: | |
| com.datastax.spark#spark-cassandra-connector_2.10;1.6.0-M1 by [com.datastax.spark#spark-cassandra-connector_2.10;1.6.2] in [default] | |
| --------------------------------------------------------------------- | |
| | | modules || artifacts | | |
| | conf | number| search|dwnlded|evicted|| number|dwnlded| | |
| --------------------------------------------------------------------- | |
| | default | 19 | 18 | 18 | 1 || 18 | 18 | | |
| --------------------------------------------------------------------- | |
| :: retrieving :: org.apache.spark#spark-submit-parent | |
| confs: [default] | |
| 18 artifacts copied, 0 already retrieved (17146kB/29ms) | |
| log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory). | |
| log4j:WARN Please initialize the log4j system properly. | |
| log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info. | |
| Using Spark's repl log4j profile: org/apache/spark/log4j-defaults-repl.properties | |
| To adjust logging level use sc.setLogLevel("INFO") | |
| Welcome to | |
| ____ __ | |
| / __/__ ___ _____/ /__ | |
| _\ \/ _ \/ _ `/ __/ '_/ | |
| /___/ .__/\_,_/_/ /_/\_\ version 1.6.1 | |
| /_/ | |
| Using Scala version 2.10.5 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_92) | |
| Type in expressions to have them evaluated. | |
| Type :help for more information. | |
| java.net.UnknownHostException: moby: moby: unknown error | |
| at java.net.InetAddress.getLocalHost(InetAddress.java:1505) | |
| at org.apache.spark.util.Utils$.findLocalInetAddress(Utils.scala:788) | |
| at org.apache.spark.util.Utils$.org$apache$spark$util$Utils$$localIpAddress$lzycompute(Utils.scala:781) | |
| at org.apache.spark.util.Utils$.org$apache$spark$util$Utils$$localIpAddress(Utils.scala:781) | |
| at org.apache.spark.util.Utils$$anonfun$localHostNameForURI$1.apply(Utils.scala:845) | |
| at org.apache.spark.util.Utils$$anonfun$localHostNameForURI$1.apply(Utils.scala:845) | |
| at scala.Option.getOrElse(Option.scala:120) | |
| at org.apache.spark.util.Utils$.localHostNameForURI(Utils.scala:845) | |
| at org.apache.spark.HttpServer.uri(HttpServer.scala:163) | |
| at org.apache.spark.repl.SparkIMain.classServerUri(SparkIMain.scala:141) | |
| at org.apache.spark.repl.SparkILoop.createSparkContext(SparkILoop.scala:1012) | |
| at $iwC$$iwC.<init>(<console>:15) | |
| at $iwC.<init>(<console>:24) | |
| at <init>(<console>:26) | |
| at .<init>(<console>:30) | |
| at .<clinit>(<console>) | |
| at .<init>(<console>:7) | |
| at .<clinit>(<console>) | |
| at $print(<console>) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1065) | |
| at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1346) | |
| at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840) | |
| at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871) | |
| at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819) | |
| at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857) | |
| at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902) | |
| at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814) | |
| at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:125) | |
| at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:124) | |
| at org.apache.spark.repl.SparkIMain.beQuietDuring(SparkIMain.scala:324) | |
| at org.apache.spark.repl.SparkILoopInit$class.initializeSpark(SparkILoopInit.scala:124) | |
| at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:64) | |
| at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1$$anonfun$apply$mcZ$sp$5.apply$mcV$sp(SparkILoop.scala:974) | |
| at org.apache.spark.repl.SparkILoopInit$class.runThunks(SparkILoopInit.scala:159) | |
| at org.apache.spark.repl.SparkILoop.runThunks(SparkILoop.scala:64) | |
| at org.apache.spark.repl.SparkILoopInit$class.postInitialization(SparkILoopInit.scala:108) | |
| at org.apache.spark.repl.SparkILoop.postInitialization(SparkILoop.scala:64) | |
| at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:991) | |
| at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945) | |
| at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945) | |
| at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135) | |
| at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945) | |
| at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1059) | |
| at org.apache.spark.repl.Main$.main(Main.scala:31) | |
| at org.apache.spark.repl.Main.main(Main.scala) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731) | |
| at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181) | |
| at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206) | |
| at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121) | |
| at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) | |
| Caused by: java.net.UnknownHostException: moby: unknown error | |
| at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method) | |
| at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928) | |
| at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323) | |
| at java.net.InetAddress.getLocalHost(InetAddress.java:1500) | |
| ... 57 more | |
| java.lang.NullPointerException | |
| at org.apache.spark.sql.SQLContext$.createListenerAndUI(SQLContext.scala:1367) | |
| at org.apache.spark.sql.hive.HiveContext.<init>(HiveContext.scala:101) | |
| at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) | |
| at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) | |
| at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) | |
| at java.lang.reflect.Constructor.newInstance(Constructor.java:423) | |
| at org.apache.spark.repl.SparkILoop.createSQLContext(SparkILoop.scala:1028) | |
| at $iwC$$iwC.<init>(<console>:15) | |
| at $iwC.<init>(<console>:24) | |
| at <init>(<console>:26) | |
| at .<init>(<console>:30) | |
| at .<clinit>(<console>) | |
| at .<init>(<console>:7) | |
| at .<clinit>(<console>) | |
| at $print(<console>) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1065) | |
| at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1346) | |
| at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840) | |
| at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871) | |
| at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819) | |
| at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857) | |
| at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902) | |
| at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814) | |
| at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:132) | |
| at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:124) | |
| at org.apache.spark.repl.SparkIMain.beQuietDuring(SparkIMain.scala:324) | |
| at org.apache.spark.repl.SparkILoopInit$class.initializeSpark(SparkILoopInit.scala:124) | |
| at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:64) | |
| at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1$$anonfun$apply$mcZ$sp$5.apply$mcV$sp(SparkILoop.scala:974) | |
| at org.apache.spark.repl.SparkILoopInit$class.runThunks(SparkILoopInit.scala:159) | |
| at org.apache.spark.repl.SparkILoop.runThunks(SparkILoop.scala:64) | |
| at org.apache.spark.repl.SparkILoopInit$class.postInitialization(SparkILoopInit.scala:108) | |
| at org.apache.spark.repl.SparkILoop.postInitialization(SparkILoop.scala:64) | |
| at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:991) | |
| at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945) | |
| at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945) | |
| at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135) | |
| at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945) | |
| at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1059) | |
| at org.apache.spark.repl.Main$.main(Main.scala:31) | |
| at org.apache.spark.repl.Main.main(Main.scala) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731) | |
| at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181) | |
| at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206) | |
| at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121) | |
| at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) | |
| <console>:16: error: not found: value sqlContext | |
| import sqlContext.implicits._ | |
| ^ | |
| <console>:16: error: not found: value sqlContext | |
| import sqlContext.sql | |
| ^ | |
| scala> import com.datastax.spark.connector._ | |
| import com.datastax.spark.connector._ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment