Merge pull request #245 from wri/lazy-open

Make tile opens be fully lazy.

src/main/scala/org/globalforestwatch/grids/Grid.scala

@@ -3,7 +3,6 @@ package org.globalforestwatch.grids
 import geotrellis.raster.TileLayout
 import geotrellis.layer.{LayoutDefinition, SpatialKey}
 import geotrellis.vector.Extent
-import org.globalforestwatch.layers.{OptionalLayer, RequiredLayer}
 
 trait Grid[T <: GridSources] {
 
@@ -56,55 +55,17 @@ trait Grid[T <: GridSources] {
     LayoutDefinition(gridExtent, tileLayout)
   }
 
+  // Get the set of grid sources (subclass of GridSources) associated with specified
+  // grid tile and the configuration/catalog in kwargs.
   def getSources(gridTile: GridTile, kwargs:  Map[String, Any]): T
 
-  def checkSources(gridTile: GridTile, windowExtent: Extent, windowKey: SpatialKey, windowLayout: LayoutDefinition, kwargs:  Map[String, Any]): T = {
-
-    def ccToMap(cc: AnyRef): Map[String, Any] =
-      cc.getClass.getDeclaredFields.foldLeft(Map.empty[String, Any]) { (a, f) =>
-        f.setAccessible(true)
-        a + (f.getName -> f.get(cc))
-      }
-
-    val sources: T = getSources(gridTile, kwargs)
-
-    val sourceMap = ccToMap(sources)
-
-    for ((k, v) <- sourceMap) {
-
-      v match {
-        case s: RequiredLayer => checkRequired(s, windowExtent)
-        case s: OptionalLayer => checkOptional(s, windowExtent)
-        case _ => Unit
-      }
-    }
-
-    sources
-  }
-
-  // NOTE: This check will cause an eager fetch of raster metadata
-  def checkRequired(layer: RequiredLayer, windowExtent: Extent): Unit = {
-    require(
-      layer.source.extent.intersects(windowExtent),
-      s"${layer.uri} does not intersect: $windowExtent"
-    )
-  }
-
-  // Only check these guys if they're defined
-  def checkOptional(layer: OptionalLayer, windowExtent: Extent): Unit = {
-    layer.source.foreach { source =>
-      require(
-        source.extent.intersects(windowExtent),
-        s"${layer.uri} does not intersect: $windowExtent"
-      )
-    }
-  }
-
+  // Get the set of grid sources (subclass of GridSources) associated with specified
+  // windowKey and windowLayout and the configuration/catalog in kwargs.
   def getRasterSource(windowKey: SpatialKey, windowLayout: LayoutDefinition, kwargs:  Map[String, Any]): T = {
     val windowExtent: Extent = windowKey.extent(windowLayout)
     val gridId = GridId.pointGridId(windowExtent.center, gridSize)
     val gridTile = GridTile(gridSize, rowCount, blockSize, gridId)
 
-    checkSources(gridTile, windowExtent: Extent, windowKey: SpatialKey, windowLayout: LayoutDefinition, kwargs:  Map[String, Any])
+    getSources(gridTile, kwargs)
   }
 }

src/main/scala/org/globalforestwatch/layers/Layer.scala

@@ -193,7 +193,9 @@ trait FLayer extends Layer {
 trait RequiredLayer extends Layer {
 
   /**
-    * Define how to read sources for required layers
+    * Define how to read sources for required layers. source is only evaluated (and
+    * so we only check the the associated tile exists) when fetchWindow is first
+    * called, and then source is used to fetch the relevant tile.
     */
   lazy val source: GDALRasterSource = {
     // Removes the expected 404 errors from console log
@@ -234,6 +236,7 @@ trait RequiredILayer extends RequiredLayer with ILayer {
     */
   def fetchWindow(windowKey: SpatialKey,
                   windowLayout: LayoutDefinition): ITile = {
+    require(source.extent.intersects(windowKey.extent(windowLayout)))
     val layoutSource = LayoutTileSource.spatial(source, windowLayout)
 //    println(s"Fetching required int tile ${source.dataPath.value}, key ${windowKey}")
     val tile = source.synchronized {
@@ -251,6 +254,7 @@ trait RequiredDLayer extends RequiredLayer with DLayer {
     */
   def fetchWindow(windowKey: SpatialKey,
                   windowLayout: LayoutDefinition): DTile = {
+    require(source.extent.intersects(windowKey.extent(windowLayout)))
     val layoutSource = LayoutTileSource.spatial(source, windowLayout)
 //    println(s"Fetching required int tile ${source.dataPath.value}, key ${windowKey}")
     val tile = source.synchronized {
@@ -268,6 +272,7 @@ trait RequiredFLayer extends RequiredLayer with FLayer {
     */
   def fetchWindow(windowKey: SpatialKey,
                   windowLayout: LayoutDefinition): FTile = {
+    require(source.extent.intersects(windowKey.extent(windowLayout)))
     val layoutSource = LayoutTileSource.spatial(source, windowLayout)
 //    println(s"Fetching required float tile ${source.dataPath.value}, key ${windowKey}")
     val tile = source.synchronized {
@@ -281,9 +286,10 @@ trait RequiredFLayer extends RequiredLayer with FLayer {
 trait OptionalLayer extends Layer {
 
   /**
-    * Define how to read sources for optional Layers
+    * Define how to read sources for optional Layers. Check if URI exists before
+    * trying to open it, return None if no file found. source is only evaluated when
+    * fetchWindow is first called, and then is used to fetch the relevant tile.
     */
-  /** Check if URI exists before trying to open it, return None if no file found */
   lazy val source: Option[GDALRasterSource] = {
     // Removes the expected 404 errors from console log
 
@@ -326,6 +332,7 @@ trait OptionalILayer extends OptionalLayer with ILayer {
     */
   def fetchWindow(windowKey: SpatialKey,
                   windowLayout: LayoutDefinition): OptionalITile = {
+    source.foreach(s => require(s.extent.intersects(windowKey.extent(windowLayout))))
 //    source.foreach(s => println(s"Fetching optional int tile ${s.dataPath.value}, key ${windowKey}"))
     new OptionalITile(for {
       source <- source
@@ -349,6 +356,7 @@ trait OptionalDLayer extends OptionalLayer with DLayer {
     */
   def fetchWindow(windowKey: SpatialKey,
                   windowLayout: LayoutDefinition): OptionalDTile = {
+    source.foreach(s => require(s.extent.intersects(windowKey.extent(windowLayout))))
 //    source.foreach(s => println(s"Fetching optional double tile ${s.dataPath.value}, key ${windowKey}"))
     new OptionalDTile(for {
       source <- source
@@ -372,6 +380,7 @@ trait OptionalFLayer extends OptionalLayer with FLayer {
     */
   def fetchWindow(windowKey: SpatialKey,
                   windowLayout: LayoutDefinition): OptionalFTile = {
+    source.foreach(s => require(s.extent.intersects(windowKey.extent(windowLayout))))
 //    source.foreach(s => println(s"Fetching optional float tile ${s.dataPath.value}, key ${windowKey}"))
     new OptionalFTile(for {
       source <- source

src/main/scala/org/globalforestwatch/summarystats/ErrorSummaryRDD.scala

@@ -190,6 +190,8 @@ trait ErrorSummaryRDD extends LazyLogging with java.io.Serializable {
     featuresGroupedWithSummaries
   }
 
+  // Get the grid sources (subclass of GridSources) associated with a specified
+  // window key and windowLayout.
   def getSources(key: SpatialKey, windowLayout: LayoutDefinition, kwargs: Map[String, Any]): Either[Throwable, SOURCES]
 
   def readWindow(rs: SOURCES, windowKey: SpatialKey, windowLayout: LayoutDefinition): Either[Throwable, Raster[TILE]]

src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala

@@ -216,6 +216,8 @@ trait SummaryRDD extends LazyLogging with java.io.Serializable {
     featuresGroupedWithSummaries
   }
 
+  // Get the grid sources (subclass of GridSources) associated with a specified
+  // window key and windowLayout.
   def getSources(key: SpatialKey, windowLayout: LayoutDefinition, kwargs: Map[String, Any]): Either[Throwable, SOURCES]
 
   def readWindow(rs: SOURCES, windowKey: SpatialKey, windowLayout: LayoutDefinition): Either[Throwable, Raster[TILE]]

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticSummary.scala

@@ -87,28 +87,18 @@ object ForestChangeDiagnosticSummary {
         val isPeatlands: Boolean = raster.tile.isPeatlands.getData(col, row)
         val isIntactForestLandscapes2000: Boolean =
           raster.tile.isIntactForestLandscapes2000.getData(col, row)
-        val prodesLossYear: Int = {
-          val loss = raster.tile.prodesLossYear.getData(col, row)
-          if (loss != null) {
-            loss.toInt
-          } else {
-            0
-          }
-        }
 
         val region: Int = raster.tile.gfwProCoverage.getData(col, row)
         val argPresence = GFWProCoverage.isArgentina(region)
         val colPresence = GFWProCoverage.isColombia(region)
         val braBiomesPresence = GFWProCoverage.isBrazilBiomesPresence(region)
-        val argOTBN: String = raster.tile.argOTBN.getData(col, row)
+        var argOTBN: String = ""
 
         // We compute country-specific forest loss using argForestLoss tile for
         // Argentina, and prodesLossYear for Brazil. In the very unusual case where a
         // location covers more than one country, we don't want to mix
         // country-specific forest losses, so we record the country-code that the
-        // forest loss came from. We will zero out the country-specific forest loss
-        // and mark it with an 'ERR' country code if we end up merging results from
-        // more than one country.
+        // forest loss came from.
         var countrySpecificLossYear = ApproxYear(0, false)
         var countryCode  = ""
         var classifiedRegion = ""
@@ -118,6 +108,7 @@ object ForestChangeDiagnosticSummary {
           classifiedRegion = raster.tile.colFronteraAgricola.getData(col, row)
         } else if (argPresence) {
           countrySpecificLossYear = raster.tile.argForestLoss.getData(col, row)
+          argOTBN = raster.tile.argOTBN.getData(col, row)
           classifiedRegion = argOTBN
           countryCode = "ARG"
         } else {
@@ -133,6 +124,20 @@ object ForestChangeDiagnosticSummary {
             countryCode = "BRA"
           }
         }
+        val prodesLossYear: Int = {
+          if (countryCode == "BRA") {
+            countrySpecificLossYear.year
+          } else {
+            0
+          }
+        }
+        val braBiomes: String = {
+          if (braBiomesPresence) {
+            raster.tile.braBiomes.getData(col, row)
+          } else {
+            ""
+          }
+        }
 
         val seAsiaLandCover: String =
           raster.tile.seAsiaLandCover.getData(col, row)
@@ -142,7 +147,6 @@ object ForestChangeDiagnosticSummary {
         val idnForestArea: String = raster.tile.idnForestArea.getData(col, row)
         val isIdnForestMoratorium: Boolean =
           raster.tile.isIDNForestMoratorium.getData(col, row)
-        val braBiomes: String = raster.tile.braBiomes.getData(col, row)
         val isPlantation: Boolean = raster.tile.isPlantation.getData(col, row)
 
         // compute Booleans