-
-
Save wnina/727734af3324d00767c6520d3edccb95 to your computer and use it in GitHub Desktop.
package tutorial | |
import geotrellis.raster._ | |
import geotrellis.raster.io.geotiff._ | |
import geotrellis.raster.render._ | |
import geotrellis.raster.resample._ | |
import geotrellis.spark._ | |
import geotrellis.spark.io._ | |
import geotrellis.spark.io.file._ | |
import geotrellis.spark.io.avro.codecs._ | |
import geotrellis.vector._ | |
import akka.actor._ | |
import akka.io.IO | |
import spray.can.Http | |
import spray.routing.{HttpService, RequestContext} | |
import spray.routing.directives.CachingDirectives | |
import spray.http.MediaTypes | |
import scala.concurrent._ | |
import com.typesafe.config.ConfigFactory | |
// more | |
import geotrellis.spark.io.hadoop._ | |
import org.apache.hadoop.fs.Path | |
import org.apache.hadoop.conf.Configuration | |
import org.apache.spark._ | |
import geotrellis.spark.util.SparkUtils | |
object ServeNDVI { | |
//val catalogPath = new java.io.File("data/catalog").getAbsolutePath | |
val inputPath = "hdfs://master:9000/datalandsat" | |
val rootPath = new Path(inputPath) | |
val hadoopConfiguration = new Configuration | |
val conf = new SparkConf() | |
.setMaster("spark://master:7077") | |
.setAppName("Spark Tunki - Serve") | |
implicit val sc = new SparkContext(conf) | |
val hadoopAttributeStore = new HadoopAttributeStore(rootPath, hadoopConfiguration) | |
// Create a reader that will read in the indexed tiles we produced in IngestImage. | |
val hadoopValueReader = HadoopValueReader(hadoopAttributeStore) | |
//val fileValueReader = FileValueReader(catalogPath) | |
def reader(layerId: LayerId) = hadoopValueReader.reader[SpatialKey, MultibandTile](layerId) | |
def main(args: Array[String]): Unit = { | |
implicit val system = akka.actor.ActorSystem("tutorial-system") | |
println(">>>>>> inside main, begin") | |
// create and start our service actor | |
val service = | |
system.actorOf(Props(classOf[NDVIServiceActor]), "tutorial") | |
// start a new HTTP server on port 8080 with our service actor as the handler | |
IO(Http) ! Http.Bind(service, "localhost", 8082) | |
println(">>>>>> inside main, end") | |
} | |
} | |
class NDVIServiceActor extends Actor with HttpService { | |
import scala.concurrent.ExecutionContext.Implicits.global | |
def actorRefFactory = context | |
def receive = runRoute(root) | |
val colorMap = | |
ColorMap.fromStringDouble(ConfigFactory.load().getString("tutorial.colormap")).get | |
def root = | |
pathPrefix(IntNumber / IntNumber / IntNumber) { (zoom, x, y) => | |
respondWithMediaType(MediaTypes.`image/png`) { | |
complete { | |
println(">>>>>> inside complete") | |
future { | |
// Read in the tile at the given z/x/y coordinates. | |
val tileOpt: Option[MultibandTile] = | |
try { | |
println(">>>> zoom: " + zoom + "x: "+ x + "y: "+ y) | |
Some(ServeNDVI.reader(LayerId("nlcd-tms",zoom)).read(x, y)) | |
} catch { | |
case _: TileNotFoundError => | |
None | |
} | |
tileOpt.map { tile => | |
// Compute the NDVI | |
val ndvi = | |
tile.convert(DoubleConstantNoDataCellType).combineDouble(0, 1) { (r, ir) => | |
if(isData(r) && isData(ir)) { | |
(ir - r) / (ir + r) | |
} else { | |
Double.NaN | |
} | |
} | |
// Render as a PNG | |
ndvi.renderPng(colorMap).bytes | |
} | |
} | |
} | |
} | |
} | |
} |
The error is:
[info] >>>>>> inside complete
[info] >>>> zoom: 8x: 76y: 138
[info] [ERROR] [05/16/2016 15:52:16.905] [ForkJoinPool-4-worker-1] [akka://tutorial-system/user/tutorial] Error during processing of request HttpRequest(GET,http://localhost:8082/8/79/138,List(Host: localhost:8082, User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:44.0) Gecko/20100101 Firefox/44.0, Accept-Language: en-US, en;q=0.5, Accept-Encoding: gzip, deflate, Connection: keep-alive, Pragma: no-cache, Cache-Control: no-cache),Empty,HTTP/1.1)
[info] java.lang.UnsupportedOperationException: empty collection
[info] at org.apache.spark.rdd.RDD$$anonfun$first$1.apply(RDD.scala:1330)
[info] at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
[info] at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111)
[info] at org.apache.spark.rdd.RDD.withScope(RDD.scala:316)
[info] at org.apache.spark.rdd.RDD.first(RDD.scala:1327)
[info] at geotrellis.spark.io.hadoop.HadoopValueReader$$anon$1.read(HadoopValueReader.scala:54)
[info] at tutorial.NDVIServiceActor$$anonfun$root$1$$anonfun$apply$1$$anonfun$apply$2.apply(ServeNDVI.scala:85)
[info] at tutorial.NDVIServiceActor$$anonfun$root$1$$anonfun$apply$1$$anonfun$apply$2.apply(ServeNDVI.scala:79)
[info] at scala.concurrent.impl.Future$PromiseCompletingRunnable.liftedTree1$1(Future.scala:24)
[info] at scala.concurrent.impl.Future$PromiseCompletingRunnable.run(Future.scala:24)
[info] at scala.concurrent.impl.ExecutionContextImpl$$anon$3.exec(ExecutionContextImpl.scala:107)
[info] at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
[info] at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
[info] at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
[info] at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
[info]
The output of json metadata (nlcd-tms___0___metadata.json and nlcd-tms___15___metadata.json) and when i ingested are:
[{"name":"nlcd-tms","zoom":0},{"header":{"format":"hdfs","keyClass":"geotrellis.spark.SpatialKey","valueClass":"geotrellis.raster.MultibandTile","path":"hdfs://master:9000/datalandsat/nlcd-tms/0"},"metadata":{"extent":{"xmin":-7931295.914632329,"ymin":-1812178.4956779396,"xmax":-7904412.241232644,"ymax":-1785130.8165442757},"layoutDefinition":{"extent":{"xmin":-20037508.342789244,"ymin":-20037508.342789244,"xmax":20037508.342789244,"ymax":20037508.342789244},"tileLayout":{"layoutCols":1,"layoutRows":1,"tileCols":256,"tileRows":256}},"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":0,"row":0}},"cellType":"int32","crs":"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs "},"keyIndex":{"type":"zorder","properties":{"keyBounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":0,"row":0}}}},"schema":{"type":"record","name":"KeyValueRecord","namespace":"geotrellis.spark.io","fields":[{"name":"pairs","type":{"type":"array","items":{"type":"record","name":"Tuple2","namespace":"scala","fields":[{"name":"_1","type":{"type":"record","name":"SpatialKey","namespace":"geotrellis.spark","fields":[{"name":"col","type":"int"},{"name":"row","type":"int"}]}},{"name":"_2","type":{"type":"record","name":"ArrayMultibandTile","namespace":"geotrellis.raster","fields":[{"name":"bands","type":{"type":"array","items":[{"type":"record","name":"ByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":-128}]},{"type":"record","name":"FloatArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"float"}},{"name":"noDataValue","type":["boolean","float"],"default":true}]},{"type":"record","name":"DoubleArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"double"}},{"name":"noDataValue","type":["boolean","double"],"default":true}]},{"type":"record","name":"ShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-32768}]},{"type":"record","name":"IntArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-2147483648}]},{"type":"record","name":"BitArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"}]},{"type":"record","name":"UByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":0}]},{"type":"record","name":"UShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":0}]}]}}]}}]}}}]}}]
[{"name":"nlcd-tms","zoom":15},{"header":{"format":"hdfs","keyClass":"geotrellis.spark.SpatialKey","valueClass":"geotrellis.raster.MultibandTile","path":"hdfs://master:9000/datalandsat/nlcd-tms/15"},"metadata":{"extent":{"xmin":-7931295.914632329,"ymin":-1812178.4956779396,"xmax":-7904412.241232644,"ymax":-1785130.8165442757},"layoutDefinition":{"extent":{"xmin":-20037508.342789244,"ymin":-20037508.342789244,"xmax":20037508.342789244,"ymax":20037508.342789244},"tileLayout":{"layoutCols":32768,"layoutRows":32768,"tileCols":256,"tileRows":256}},"bounds":{"minKey":{"col":9898,"row":17843},"maxKey":{"col":9920,"row":17865}},"cellType":"int32","crs":"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs "},"keyIndex":{"type":"zorder","properties":{"keyBounds":{"minKey":{"col":9898,"row":17843},"maxKey":{"col":9920,"row":17865}}}},"schema":{"type":"record","name":"KeyValueRecord","namespace":"geotrellis.spark.io","fields":[{"name":"pairs","type":{"type":"array","items":{"type":"record","name":"Tuple2","namespace":"scala","fields":[{"name":"_1","type":{"type":"record","name":"SpatialKey","namespace":"geotrellis.spark","fields":[{"name":"col","type":"int"},{"name":"row","type":"int"}]}},{"name":"_2","type":{"type":"record","name":"ArrayMultibandTile","namespace":"geotrellis.raster","fields":[{"name":"bands","type":{"type":"array","items":[{"type":"record","name":"ByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":-128}]},{"type":"record","name":"FloatArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"float"}},{"name":"noDataValue","type":["boolean","float"],"default":true}]},{"type":"record","name":"DoubleArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"double"}},{"name":"noDataValue","type":["boolean","double"],"default":true}]},{"type":"record","name":"ShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-32768}]},{"type":"record","name":"IntArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-2147483648}]},{"type":"record","name":"BitArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"}]},{"type":"record","name":"UByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":0}]},{"type":"record","name":"UShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":0}]}]}}]}}]}}}]}}]
The script that use to ingest was: