Only load detailed wdpa tile, rename the wdpa fields

From Justin's comments: only open and load the protectedAreasByCategory tiles, since the isProtected property is easily determined from the protected area category, so we don't need to load the other protected areas dataset. Also, rename wdpa_area field to protected_areas_by_category, wdpa/detailedProtectedAreas variables to protectedAreasByCategory, etc.
wri · Sep 26, 2023 · bec3fdb · bec3fdb
1 parent cbb89ac
commit bec3fdb
Show file tree

Hide file tree

Showing 11 changed files with 22 additions and 29 deletions.
diff --git a/src/main/scala/org/globalforestwatch/layers/DetailedProtectedAreas.scala b/src/main/scala/org/globalforestwatch/layers/DetailedProtectedAreas.scala
@@ -20,6 +20,6 @@ case class DetailedProtectedAreas(gridTile: GridTile, kwargs: Map[String, Any])
     case 11 => "Not Reported"
     case 12 => "Not Applicable"
     case 13 => "Not Assigned"
-    case _  => "Unknown"
+    case _  => ""
   }
 }
diff --git a/src/main/scala/org/globalforestwatch/layers/Layer.scala b/src/main/scala/org/globalforestwatch/layers/Layer.scala
@@ -237,7 +237,6 @@ trait RequiredILayer extends RequiredLayer with ILayer {
   def fetchWindow(windowKey: SpatialKey,
                   windowLayout: LayoutDefinition): ITile = {
     val layoutSource = LayoutTileSource.spatial(source, windowLayout)
-    println(s"Fetching required int tile ${source.dataPath.value}, key ${windowKey}")
     val tile = source.synchronized {
       layoutSource.read(windowKey).get.band(0)
     }
@@ -254,7 +253,6 @@ trait RequiredDLayer extends RequiredLayer with DLayer {
   def fetchWindow(windowKey: SpatialKey,
                   windowLayout: LayoutDefinition): DTile = {
     val layoutSource = LayoutTileSource.spatial(source, windowLayout)
-    println(s"Fetching required double tile ${source.dataPath.value}, key ${windowKey}")
     val tile = source.synchronized {
       layoutSource.read(windowKey).get.band(0)
     }
@@ -271,7 +269,6 @@ trait RequiredFLayer extends RequiredLayer with FLayer {
   def fetchWindow(windowKey: SpatialKey,
                   windowLayout: LayoutDefinition): FTile = {
     val layoutSource = LayoutTileSource.spatial(source, windowLayout)
-    println(s"Fetching required float tile ${source.dataPath.value}, key ${windowKey}")
     val tile = source.synchronized {
       layoutSource.read(windowKey).get.band(0)
     }
@@ -328,7 +325,6 @@ trait OptionalILayer extends OptionalLayer with ILayer {
     */
   def fetchWindow(windowKey: SpatialKey,
                   windowLayout: LayoutDefinition): OptionalITile = {
-    source.foreach(s => println(s"Fetching optional int tile ${s.dataPath.value}, key ${windowKey}"))
     new OptionalITile(for {
       source <- source
       raster <- Either
@@ -350,8 +346,7 @@ trait OptionalDLayer extends OptionalLayer with DLayer {
     * Define how to fetch data for optional double rasters
     */
   def fetchWindow(windowKey: SpatialKey,
-                  windowLayout: LayoutDefinition): OptionalDTile = {
-    source.foreach(s => println(s"Fetching optional double tile ${s.dataPath.value}, key ${windowKey}"))
+                  windowLayout: LayoutDefinition): OptionalDTile =
     new OptionalDTile(for {
       source <- source
       raster <- Either
@@ -364,7 +359,6 @@ trait OptionalDLayer extends OptionalLayer with DLayer {
         })
         .toOption
     } yield raster)
-  }
 }
 
 trait OptionalFLayer extends OptionalLayer with FLayer {
@@ -373,8 +367,7 @@ trait OptionalFLayer extends OptionalLayer with FLayer {
     * Define how to fetch data for optional double rasters
     */
   def fetchWindow(windowKey: SpatialKey,
-                  windowLayout: LayoutDefinition): OptionalFTile = {
-    source.foreach(s => println(s"Fetching optional float tile ${s.dataPath.value}, key ${windowKey}"))
+                  windowLayout: LayoutDefinition): OptionalFTile =
     new OptionalFTile(for {
       source <- source
       raster <- Either
@@ -387,7 +380,6 @@ trait OptionalFLayer extends OptionalLayer with FLayer {
         })
         .toOption
     } yield raster)
-  }
 }
 
 trait BooleanLayer extends ILayer {

diff --git a/...balforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala b/...balforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala
@@ -42,9 +42,6 @@ object ForestChangeDiagnosticAnalysis extends SummaryAnalysis {
     kwargs: Map[String, Any]
   )(implicit spark: SparkSession): RDD[ValidatedLocation[ForestChangeDiagnosticData]] = {
     features.persist(StorageLevel.MEMORY_AND_DISK)
-    // For debugging - will be removed before checkin.
-    println(s"Number of rows: ${features.collect().length}")
-    features.collect().foreach(println)
 
     try {
       val diffGridIds: List[GridId] =

diff --git a/...rg/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDF.scala b/...rg/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDF.scala
@@ -116,7 +116,7 @@ object ForestChangeDiagnosticDF extends SummaryDF {
     "protected_areas_area", // protectedAreasArea
     "peat_area", // peatlandsArea
     "arg_otbn_area", // argOTBNArea
-    "wdpa_area", // detailedProtectedAreas
+    "protected_areas_by_category_area", // protectedAreasByCategory
     "landmark_area", // landmarkArea
     "brazil_biomes", // braBiomesArea
     "idn_legal_area", // idnForestAreaArea

diff --git a/.../globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala b/.../globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala
@@ -48,7 +48,7 @@ case class ForestChangeDiagnosticData(
   /** OTBN category area */
   arg_otbn_area: ForestChangeDiagnosticDataDoubleCategory,
   /** Detailed WDPA category area */
-  wdpa_area: ForestChangeDiagnosticDataDoubleCategory,
+  protected_areas_by_category_area: ForestChangeDiagnosticDataDoubleCategory,
   /** Indigenous area (from Landmark dataset) */
   landmark_area: ForestChangeDiagnosticDataDouble,
   brazil_biomes: ForestChangeDiagnosticDataDoubleCategory,
@@ -139,7 +139,7 @@ case class ForestChangeDiagnosticData(
       protected_areas_area.merge(other.protected_areas_area),
       peat_area.merge(other.peat_area),
       arg_otbn_area.merge(other.arg_otbn_area),
-      wdpa_area.merge(other.wdpa_area),
+      protected_areas_by_category_area.merge(other.protected_areas_by_category_area),
       landmark_area.merge(other.landmark_area),
       brazil_biomes.merge(other.brazil_biomes),
       idn_legal_area.merge(other.idn_legal_area),

diff --git a/...forestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticGridSources.scala b/...forestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticGridSources.scala
@@ -17,7 +17,6 @@ case class ForestChangeDiagnosticGridSources(gridTile: GridTile, kwargs: Map[Str
   val isPrimaryForest: PrimaryForest = PrimaryForest(gridTile, kwargs)
   val isPeatlands: Peatlands = Peatlands(gridTile, kwargs)
   val isIntactForestLandscapes2000: IntactForestLandscapes2000 = IntactForestLandscapes2000(gridTile, kwargs)
-  val protectedAreas: ProtectedAreas = ProtectedAreas(gridTile, kwargs)
   val seAsiaLandCover: SEAsiaLandCover = SEAsiaLandCover(gridTile, kwargs)
   val idnLandCover: IndonesiaLandCover = IndonesiaLandCover(gridTile, kwargs)
   val isSoyPlantedArea: SoyPlantedAreas = SoyPlantedAreas(gridTile, kwargs)
@@ -28,7 +27,7 @@ case class ForestChangeDiagnosticGridSources(gridTile: GridTile, kwargs: Map[Str
   val isPlantation: PlantedForestsBool = PlantedForestsBool(gridTile, kwargs)
   val gfwProCoverage: GFWProCoverage = GFWProCoverage(gridTile, kwargs)
   val argOTBN: ArgOTBN = ArgOTBN(gridTile, kwargs)
-  val detailedProtectedAreas: DetailedProtectedAreas = DetailedProtectedAreas(gridTile, kwargs)
+  val protectedAreasByCategory: DetailedProtectedAreas = DetailedProtectedAreas(gridTile, kwargs)
   val landmark: Landmark = Landmark(gridTile, kwargs)
 
 

diff --git a/...orestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawDataGroup.scala b/...orestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawDataGroup.scala
@@ -25,7 +25,7 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int,
                                               seAsiaPresence: Boolean,
                                               idnPresence: Boolean,
                                               argPresence: Boolean,
-                                              wdpa: String,
+                                              protectedAreaByCategory: String,
                                               landmark: Boolean,
 ) {
 
@@ -158,8 +158,8 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int,
       .fill(totalArea, isPeatlands),
     arg_otbn_area = ForestChangeDiagnosticDataDoubleCategory
       .fill(argOTBN, totalArea),
-    wdpa_area = ForestChangeDiagnosticDataDoubleCategory
-      .fill(wdpa, totalArea),
+    protected_areas_by_category_area = ForestChangeDiagnosticDataDoubleCategory
+      .fill(protectedAreaByCategory, totalArea),
     landmark_area = ForestChangeDiagnosticDataDouble.fill(totalArea, landmark),
     brazil_biomes = ForestChangeDiagnosticDataDoubleCategory
       .fill(braBiomes, totalArea),

diff --git a/...obalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticSummary.scala b/...obalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticSummary.scala
@@ -75,7 +75,6 @@ object ForestChangeDiagnosticSummary {
         val isPeatlands: Boolean = raster.tile.isPeatlands.getData(col, row)
         val isIntactForestLandscapes2000: Boolean =
           raster.tile.isIntactForestLandscapes2000.getData(col, row)
-        val wdpa: String = raster.tile.wdpaProtectedAreas.getData(col, row)
         val prodesLossYear: Int = {
           val loss = raster.tile.prodesLossYear.getData(col, row)
           if (loss != null) {
@@ -102,7 +101,6 @@ object ForestChangeDiagnosticSummary {
         val isTreeCoverExtent30: Boolean = tcd2000 > 30
         val isTreeCoverExtent90: Boolean = tcd2000 > 90
         val isUMDLoss: Boolean = isTreeCoverExtent30 && umdTreeCoverLossYear > 0
-        val isProtectedArea: Boolean = wdpa != ""
         val isProdesLoss: Boolean = prodesLossYear > 0
 
         val southAmericaPresence =
@@ -116,13 +114,16 @@ object ForestChangeDiagnosticSummary {
         val idnPresence = gfwProCoverage.getOrElse("Indonesia", false)
         val argPresence = gfwProCoverage.getOrElse("Argentina", false)
 
+        val protectedAreaCategory = raster.tile.protectedAreasByCategory.getData(col, row)
+        val isProtectedArea = (protectedAreaCategory != "")
+
         // Currently, only do the area intersection with the detailed WDPA categories
         // if location is in Argentina. Similarly, only do area intersection with
         // Landmark (indigenous territories) if in Argentina.
-        // With lazy tile loading, the WDPA and landmark tiles are only loaded if
+        // With lazy tile loading, the landmark tiles are only loaded if
         // argPresence is true.
         val detailedWdpa = if (argPresence)
-          raster.tile.detailedWdpaProtectedAreas.getData(col, row)
+          protectedAreaCategory
         else
           ""
         val landmark = if (argPresence)

diff --git a/.../globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticTile.scala b/.../globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticTile.scala
@@ -20,7 +20,6 @@ case class ForestChangeDiagnosticTile(
   lazy val isPeatlands = sources.isPeatlands.fetchWindow(windowKey, windowLayout)
   lazy val isIntactForestLandscapes2000 =
         sources.isIntactForestLandscapes2000.fetchWindow(windowKey, windowLayout)
-  lazy val wdpaProtectedAreas = sources.protectedAreas.fetchWindow(windowKey, windowLayout)
   lazy val seAsiaLandCover = sources.seAsiaLandCover.fetchWindow(windowKey, windowLayout)
   lazy val idnLandCover = sources.idnLandCover.fetchWindow(windowKey, windowLayout)
   lazy val isSoyPlantedArea = sources.isSoyPlantedArea.fetchWindow(windowKey, windowLayout)
@@ -32,7 +31,7 @@ case class ForestChangeDiagnosticTile(
   lazy val gfwProCoverage = sources.gfwProCoverage.fetchWindow(windowKey, windowLayout)
   lazy val argOTBN = sources.argOTBN.fetchWindow(windowKey, windowLayout)
 
-  lazy val detailedWdpaProtectedAreas = sources.detailedProtectedAreas.fetchWindow(windowKey, windowLayout)
+  lazy val protectedAreasByCategory = sources.protectedAreasByCategory.fetchWindow(windowKey, windowLayout)
   lazy val landmark = sources.landmark.fetchWindow(windowKey, windowLayout)
 
   def cellType: CellType = loss.cellType