-
-
Save wnina/727734af3324d00767c6520d3edccb95 to your computer and use it in GitHub Desktop.
| package tutorial | |
| import geotrellis.raster._ | |
| import geotrellis.raster.io.geotiff._ | |
| import geotrellis.raster.render._ | |
| import geotrellis.raster.resample._ | |
| import geotrellis.spark._ | |
| import geotrellis.spark.io._ | |
| import geotrellis.spark.io.file._ | |
| import geotrellis.spark.io.avro.codecs._ | |
| import geotrellis.vector._ | |
| import akka.actor._ | |
| import akka.io.IO | |
| import spray.can.Http | |
| import spray.routing.{HttpService, RequestContext} | |
| import spray.routing.directives.CachingDirectives | |
| import spray.http.MediaTypes | |
| import scala.concurrent._ | |
| import com.typesafe.config.ConfigFactory | |
| // more | |
| import geotrellis.spark.io.hadoop._ | |
| import org.apache.hadoop.fs.Path | |
| import org.apache.hadoop.conf.Configuration | |
| import org.apache.spark._ | |
| import geotrellis.spark.util.SparkUtils | |
| object ServeNDVI { | |
| //val catalogPath = new java.io.File("data/catalog").getAbsolutePath | |
| val inputPath = "hdfs://master:9000/datalandsat" | |
| val rootPath = new Path(inputPath) | |
| val hadoopConfiguration = new Configuration | |
| val conf = new SparkConf() | |
| .setMaster("spark://master:7077") | |
| .setAppName("Spark Tunki - Serve") | |
| implicit val sc = new SparkContext(conf) | |
| val hadoopAttributeStore = new HadoopAttributeStore(rootPath, hadoopConfiguration) | |
| // Create a reader that will read in the indexed tiles we produced in IngestImage. | |
| val hadoopValueReader = HadoopValueReader(hadoopAttributeStore) | |
| //val fileValueReader = FileValueReader(catalogPath) | |
| def reader(layerId: LayerId) = hadoopValueReader.reader[SpatialKey, MultibandTile](layerId) | |
| def main(args: Array[String]): Unit = { | |
| implicit val system = akka.actor.ActorSystem("tutorial-system") | |
| println(">>>>>> inside main, begin") | |
| // create and start our service actor | |
| val service = | |
| system.actorOf(Props(classOf[NDVIServiceActor]), "tutorial") | |
| // start a new HTTP server on port 8080 with our service actor as the handler | |
| IO(Http) ! Http.Bind(service, "localhost", 8082) | |
| println(">>>>>> inside main, end") | |
| } | |
| } | |
| class NDVIServiceActor extends Actor with HttpService { | |
| import scala.concurrent.ExecutionContext.Implicits.global | |
| def actorRefFactory = context | |
| def receive = runRoute(root) | |
| val colorMap = | |
| ColorMap.fromStringDouble(ConfigFactory.load().getString("tutorial.colormap")).get | |
| def root = | |
| pathPrefix(IntNumber / IntNumber / IntNumber) { (zoom, x, y) => | |
| respondWithMediaType(MediaTypes.`image/png`) { | |
| complete { | |
| println(">>>>>> inside complete") | |
| future { | |
| // Read in the tile at the given z/x/y coordinates. | |
| val tileOpt: Option[MultibandTile] = | |
| try { | |
| println(">>>> zoom: " + zoom + "x: "+ x + "y: "+ y) | |
| Some(ServeNDVI.reader(LayerId("nlcd-tms",zoom)).read(x, y)) | |
| } catch { | |
| case _: TileNotFoundError => | |
| None | |
| } | |
| tileOpt.map { tile => | |
| // Compute the NDVI | |
| val ndvi = | |
| tile.convert(DoubleConstantNoDataCellType).combineDouble(0, 1) { (r, ir) => | |
| if(isData(r) && isData(ir)) { | |
| (ir - r) / (ir + r) | |
| } else { | |
| Double.NaN | |
| } | |
| } | |
| // Render as a PNG | |
| ndvi.renderPng(colorMap).bytes | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } |
The error is:
[info] >>>>>> inside complete
[info] >>>> zoom: 8x: 76y: 138
[info] [ERROR] [05/16/2016 15:52:16.905] [ForkJoinPool-4-worker-1] [akka://tutorial-system/user/tutorial] Error during processing of request HttpRequest(GET,http://localhost:8082/8/79/138,List(Host: localhost:8082, User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:44.0) Gecko/20100101 Firefox/44.0, Accept-Language: en-US, en;q=0.5, Accept-Encoding: gzip, deflate, Connection: keep-alive, Pragma: no-cache, Cache-Control: no-cache),Empty,HTTP/1.1)
[info] java.lang.UnsupportedOperationException: empty collection
[info] at org.apache.spark.rdd.RDD$$anonfun$first$1.apply(RDD.scala:1330)
[info] at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
[info] at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111)
[info] at org.apache.spark.rdd.RDD.withScope(RDD.scala:316)
[info] at org.apache.spark.rdd.RDD.first(RDD.scala:1327)
[info] at geotrellis.spark.io.hadoop.HadoopValueReader$$anon$1.read(HadoopValueReader.scala:54)
[info] at tutorial.NDVIServiceActor$$anonfun$root$1$$anonfun$apply$1$$anonfun$apply$2.apply(ServeNDVI.scala:85)
[info] at tutorial.NDVIServiceActor$$anonfun$root$1$$anonfun$apply$1$$anonfun$apply$2.apply(ServeNDVI.scala:79)
[info] at scala.concurrent.impl.Future$PromiseCompletingRunnable.liftedTree1$1(Future.scala:24)
[info] at scala.concurrent.impl.Future$PromiseCompletingRunnable.run(Future.scala:24)
[info] at scala.concurrent.impl.ExecutionContextImpl$$anon$3.exec(ExecutionContextImpl.scala:107)
[info] at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
[info] at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
[info] at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
[info] at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
[info]
The output of json metadata (nlcd-tms___0___metadata.json and nlcd-tms___15___metadata.json) and when i ingested are:
[{"name":"nlcd-tms","zoom":0},{"header":{"format":"hdfs","keyClass":"geotrellis.spark.SpatialKey","valueClass":"geotrellis.raster.MultibandTile","path":"hdfs://master:9000/datalandsat/nlcd-tms/0"},"metadata":{"extent":{"xmin":-7931295.914632329,"ymin":-1812178.4956779396,"xmax":-7904412.241232644,"ymax":-1785130.8165442757},"layoutDefinition":{"extent":{"xmin":-20037508.342789244,"ymin":-20037508.342789244,"xmax":20037508.342789244,"ymax":20037508.342789244},"tileLayout":{"layoutCols":1,"layoutRows":1,"tileCols":256,"tileRows":256}},"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":0,"row":0}},"cellType":"int32","crs":"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs "},"keyIndex":{"type":"zorder","properties":{"keyBounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":0,"row":0}}}},"schema":{"type":"record","name":"KeyValueRecord","namespace":"geotrellis.spark.io","fields":[{"name":"pairs","type":{"type":"array","items":{"type":"record","name":"Tuple2","namespace":"scala","fields":[{"name":"_1","type":{"type":"record","name":"SpatialKey","namespace":"geotrellis.spark","fields":[{"name":"col","type":"int"},{"name":"row","type":"int"}]}},{"name":"_2","type":{"type":"record","name":"ArrayMultibandTile","namespace":"geotrellis.raster","fields":[{"name":"bands","type":{"type":"array","items":[{"type":"record","name":"ByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":-128}]},{"type":"record","name":"FloatArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"float"}},{"name":"noDataValue","type":["boolean","float"],"default":true}]},{"type":"record","name":"DoubleArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"double"}},{"name":"noDataValue","type":["boolean","double"],"default":true}]},{"type":"record","name":"ShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-32768}]},{"type":"record","name":"IntArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-2147483648}]},{"type":"record","name":"BitArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"}]},{"type":"record","name":"UByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":0}]},{"type":"record","name":"UShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":0}]}]}}]}}]}}}]}}]
[{"name":"nlcd-tms","zoom":15},{"header":{"format":"hdfs","keyClass":"geotrellis.spark.SpatialKey","valueClass":"geotrellis.raster.MultibandTile","path":"hdfs://master:9000/datalandsat/nlcd-tms/15"},"metadata":{"extent":{"xmin":-7931295.914632329,"ymin":-1812178.4956779396,"xmax":-7904412.241232644,"ymax":-1785130.8165442757},"layoutDefinition":{"extent":{"xmin":-20037508.342789244,"ymin":-20037508.342789244,"xmax":20037508.342789244,"ymax":20037508.342789244},"tileLayout":{"layoutCols":32768,"layoutRows":32768,"tileCols":256,"tileRows":256}},"bounds":{"minKey":{"col":9898,"row":17843},"maxKey":{"col":9920,"row":17865}},"cellType":"int32","crs":"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs "},"keyIndex":{"type":"zorder","properties":{"keyBounds":{"minKey":{"col":9898,"row":17843},"maxKey":{"col":9920,"row":17865}}}},"schema":{"type":"record","name":"KeyValueRecord","namespace":"geotrellis.spark.io","fields":[{"name":"pairs","type":{"type":"array","items":{"type":"record","name":"Tuple2","namespace":"scala","fields":[{"name":"_1","type":{"type":"record","name":"SpatialKey","namespace":"geotrellis.spark","fields":[{"name":"col","type":"int"},{"name":"row","type":"int"}]}},{"name":"_2","type":{"type":"record","name":"ArrayMultibandTile","namespace":"geotrellis.raster","fields":[{"name":"bands","type":{"type":"array","items":[{"type":"record","name":"ByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":-128}]},{"type":"record","name":"FloatArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"float"}},{"name":"noDataValue","type":["boolean","float"],"default":true}]},{"type":"record","name":"DoubleArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"double"}},{"name":"noDataValue","type":["boolean","double"],"default":true}]},{"type":"record","name":"ShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-32768}]},{"type":"record","name":"IntArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-2147483648}]},{"type":"record","name":"BitArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"}]},{"type":"record","name":"UByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":0}]},{"type":"record","name":"UShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":0}]}]}}]}}]}}}]}}]
The script that use to ingest was: