Created
October 4, 2018 03:04
-
-
Save ianmilligan1/70ea619752c8e77d497b690504fe806b to your computer and use it in GitHub Desktop.
errorlog.txt
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| scala> :paste | |
| // Entering paste mode (ctrl-D to finish) | |
| import io.archivesunleashed._ | |
| import io.archivesunleashed.app._ | |
| import io.archivesunleashed.matchbox._ | |
| sc.setLogLevel("INFO") | |
| RecordLoader.loadArchives("/mnt/vol1/data_sets/aut_debug/issue-499/", sc).keepValidPages().map(r => ExtractDomain(r.getUrl)).countItems().saveAsTextFile("/mnt/vol1/data_sets/aut_debug/issue-499/results/domains") | |
| RecordLoader.loadArchives("/mnt/vol1/data_sets/aut_debug/issue-499/", sc).keepValidPages().map(r => (r.getCrawlDate, r.getDomain, r.getUrl, RemoveHTML(r.getContentString))).saveAsTextFile("/mnt/vol1/data_sets/aut_debug/issue-499/results/test") | |
| val links = RecordLoader.loadArchives("/mnt/vol1/data_sets/aut_debug/issue-499/", sc).keepValidPages().map(r => (r.getCrawlDate, ExtractLinks(r.getUrl, r.getContentString))).flatMap(r => r._2.map(f => (r._1, ExtractDomain(f._1).replaceAll("^\\s*www\\.", ""), ExtractDomain(f._2).replaceAll("^\\s*www\\.", "")))).filter(r => r._2 != "" && r._3 != "").countItems().filter(r => r._2 > 5) | |
| WriteGraphML(links, "/mnt/vol1/data_sets/aut_debug/issue-499/results/499-gephi.graphml") | |
| sys.exit | |
| // Exiting paste mode, now interpreting. | |
| 2018-10-03 23:03:58,667 [main] INFO MemoryStore - Block broadcast_0 stored as values in memory (estimated size 221.8 KB, free 10.5 GB) | |
| 2018-10-03 23:03:58,743 [main] INFO MemoryStore - Block broadcast_0_piece0 stored as bytes in memory (estimated size 22.2 KB, free 10.5 GB) | |
| 2018-10-03 23:03:58,745 [dispatcher-event-loop-4] INFO BlockManagerInfo - Added broadcast_0_piece0 in memory on 130.63.180.18:33209 (size: 22.2 KB, free: 10.5 GB) | |
| 2018-10-03 23:03:58,752 [main] INFO SparkContext - Created broadcast 0 from newAPIHadoopFile at package.scala:66 | |
| 2018-10-03 23:03:58,924 [main] INFO FileInputFormat - Total input paths to process : 67 | |
| java.lang.NullPointerException | |
| at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getSplits(FileInputFormat.java:417) | |
| at org.apache.spark.rdd.NewHadoopRDD.getPartitions(NewHadoopRDD.scala:125) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250) | |
| at scala.Option.getOrElse(Option.scala:121) | |
| at org.apache.spark.rdd.RDD.partitions(RDD.scala:250) | |
| at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250) | |
| at scala.Option.getOrElse(Option.scala:121) | |
| at org.apache.spark.rdd.RDD.partitions(RDD.scala:250) | |
| at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250) | |
| at scala.Option.getOrElse(Option.scala:121) | |
| at org.apache.spark.rdd.RDD.partitions(RDD.scala:250) | |
| at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250) | |
| at scala.Option.getOrElse(Option.scala:121) | |
| at org.apache.spark.rdd.RDD.partitions(RDD.scala:250) | |
| at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250) | |
| at scala.Option.getOrElse(Option.scala:121) | |
| at org.apache.spark.rdd.RDD.partitions(RDD.scala:250) | |
| at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252) | |
| at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250) | |
| at scala.Option.getOrElse(Option.scala:121) | |
| at org.apache.spark.rdd.RDD.partitions(RDD.scala:250) | |
| at org.apache.spark.Partitioner$$anonfun$defaultPartitioner$2.apply(Partitioner.scala:66) | |
| at org.apache.spark.Partitioner$$anonfun$defaultPartitioner$2.apply(Partitioner.scala:66) | |
| at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) | |
| at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) | |
| at scala.collection.immutable.List.foreach(List.scala:381) | |
| at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) | |
| at scala.collection.immutable.List.map(List.scala:285) | |
| at org.apache.spark.Partitioner$.defaultPartitioner(Partitioner.scala:66) | |
| at org.apache.spark.rdd.PairRDDFunctions$$anonfun$reduceByKey$3.apply(PairRDDFunctions.scala:331) | |
| at org.apache.spark.rdd.PairRDDFunctions$$anonfun$reduceByKey$3.apply(PairRDDFunctions.scala:331) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) | |
| at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) | |
| at org.apache.spark.rdd.PairRDDFunctions.reduceByKey(PairRDDFunctions.scala:330) | |
| at io.archivesunleashed.package$CountableRDD.countItems(package.scala:89) | |
| ... 53 elided | |
| scala> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment