Merge pull request #238 from wri/gtc-2826

GTC-2826 Colombia-specific analysis in forest_change_diagnostic

.gitignore

@@ -54,4 +54,3 @@ nohup.out
 derby.log
 metastore_db/
 *.log
-*.tsv

src/main/resources/raster-catalog-pro.json

@@ -222,7 +222,7 @@
     },
     {
       "name":"gfwpro_forest_change_regions",
-      "source_uri": "s3://gfw-data-lake/gfwpro_forest_change_regions/v20230207/raster/epsg-4326/{grid_size}/{row_count}/bit_encoding/gdal-geotiff/{tile_id}.tif"
+      "source_uri": "s3://gfw-data-lake/gfwpro_forest_change_regions/v20240529/raster/epsg-4326/{grid_size}/{row_count}/bit_encoding/gdal-geotiff/{tile_id}.tif"
     },
     {
       "name":"ifl_intact_forest_landscapes_2000",
@@ -263,6 +263,10 @@
     {
       "name":"jrc_global_forest_cover",
       "source_uri":"s3://gfw-data-lake/jrc_global_forest_cover/v2020/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"col_frontera_agricola",
+      "source_uri":"s3://gfw-data-lake/col_frontera_agricola/v2024/raster/epsg-4326/{grid_size}/{row_count}/category/gdal-geotiff/{tile_id}.tif"
     }
   ]
 }

src/main/scala/org/globalforestwatch/layers/ColFronteraAgricola.scala

@@ -0,0 +1,27 @@
+package org.globalforestwatch.layers
+
+import org.globalforestwatch.grids.GridTile
+
+// Colombia classification of all its lands into areas where agricultural activity is
+// allowed, or is conditional, or is fully restricted. The classification strings are
+// intended to be parsed and used directly by the front-end (so front end doesn't
+// have to have country-specific information in it).
+case class ColFronteraAgricola(gridTile: GridTile, kwargs: Map[String, Any]) extends StringLayer with OptionalILayer {
+  val datasetName = "col_frontera_agricola"
+  val uri: String = uriForGrid(gridTile, kwargs)
+
+  def lookup(value: Int): String = value match {
+    case 1 => "No condicionado;Frontera agricola nacional"
+    case 2 => "Condicionado;Ambiental"
+    case 3 => "Condicionado;Ambiental/Riesgo de desastres"
+    case 4 => "Condicionado;Ambiental/Riesgo de desastres/Étnico-Cultural'"
+    case 5 => "Condicionado;Ambiental/Étnico-Cultural"
+    case 6 => "Condicionado;Gestión riesgo de desastres"
+    case 7 => "Condicionado;Riesgo de desastres/Étnico-Cultural"
+    case 8 => "Condicionado;Étnico-Cultural"
+    case 9 => "Zona restricción;Restricciones acuerdo cero deforestación"
+    case 10 => "Zona restricción;Restricciones legales"
+    case 11 => "Zona restricción;Restricciones técnicas (Áreas no agropecuarias)"
+    case _  => ""
+  }
+}

src/main/scala/org/globalforestwatch/layers/GFWProCoverage.scala

@@ -3,25 +3,24 @@ package org.globalforestwatch.layers
 import org.globalforestwatch.grids.GridTile
 
 case class GFWProCoverage(gridTile: GridTile, kwargs: Map[String, Any])
-  extends MapILayer
+  extends IntegerLayer
     with OptionalILayer {
 
   val datasetName = "gfwpro_forest_change_regions"
 
   val uri: String =
     uriForGrid(gridTile, kwargs)
+}
 
-  def lookup(value: Int): Map[String, Boolean] = {
-    val bits = "000000000" + value.toBinaryString takeRight 9
-    Map(
-      "Argentina" -> (bits(8) == '1'),
-      "Soy Coverage" -> (bits(7) == '1'),
-      "South America" -> (bits(6) == '1'),
-      "Legal Amazon" -> (bits(5) == '1'),
-      "Brazil Biomes" -> (bits(4) == '1'),
-      "Cerrado Biomes" -> (bits(3) == '1'),
-      "South East Asia" -> (bits(2) == '1'),
-      "Indonesia" -> (bits(1) == '1')
-    )
-  }
+object GFWProCoverage {
+  def isArgentina(v: Integer): Boolean = (v & (1 << 0)) != 0
+  def isColombia(v: Integer): Boolean  = (v & (1 << 1)) != 0
+  def isSouthAmerica(v: Integer): Boolean  = (v & (1 << 2)) != 0
+  def isLegalAmazonPresence(v: Integer): Boolean  = (v & (1 << 3)) != 0
+  // Brazil Biomes presence is true for all Brazil, except for a few narrow costal
+  // areas.
+  def isBrazilBiomesPresence(v: Integer): Boolean  = (v & (1 << 4)) != 0
+  def isCerradoBiomesPresence(v: Integer): Boolean  = (v & (1 << 5)) != 0
+  def isSouthEastAsia(v: Integer): Boolean  = (v & (1 << 6)) != 0
+  def isIndonesia(v: Integer): Boolean  = (v & (1 << 7)) != 0
 }

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDF.scala

@@ -98,6 +98,7 @@ object ForestChangeDiagnosticDF extends SummaryDF {
     "tree_cover_loss_by_country_yearly",
     "tree_cover_loss_by_country_wdpa_yearly",
     "tree_cover_loss_by_country_landmark_yearly",
+    "tree_cover_loss_by_country_classified_region_yearly",
     "tree_cover_loss_arg_otbn_yearly", // treeCoverLossARGOTBNYearly
     "tree_cover_loss_sea_landcover_yearly", // treeCoverLossSEAsiaLandCoverYearly
     "tree_cover_loss_idn_landcover_yearly", // treeCoverLossIDNLandCoverYearly
@@ -107,10 +108,12 @@ object ForestChangeDiagnosticDF extends SummaryDF {
     "tree_cover_loss_prodes_yearly", // prodesLossYearly
     "tree_cover_loss_prodes_wdpa_yearly", // prodesLossProtectedAreasYearly
     "tree_cover_loss_prodes_primary_forest_yearly", // prodesLossProdesPrimaryForestYearly
+    "country_area",
     "country_specific_deforestation_yearly",
     "country_specific_deforestation_wdpa_yearly",
     "country_specific_deforestation_landmark_yearly",
     "country_specific_deforestation_classified_region_yearly",
+    "classified_region_area",
     "tree_cover_loss_brazil_biomes_yearly", // treeCoverLossBRABiomesYearly
     "tree_cover_extent_total", // treeCoverExtent
     "tree_cover_extent_primary_forest", // treeCoverExtentPrimaryForest

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala

@@ -18,11 +18,14 @@ case class ForestChangeDiagnosticData(
   tree_cover_loss_intact_forest_yearly: ForestChangeDiagnosticDataLossYearly,
   tree_cover_loss_protected_areas_yearly: ForestChangeDiagnosticDataLossYearly,
   // The "tree_cover_loss_by_country_*" fields are only filled in per-country when
-  // countryCode != "" (currently only ARG and BRA). These are for umdLoss
+  // countryCode != "" (currently only ARG, BRA, and COL). These are for umdLoss
   // comparison bars within the country-specific UI tabs.
   tree_cover_loss_by_country_yearly: ForestChangeDiagnosticDataLossYearlyCategory,
   tree_cover_loss_by_country_wdpa_yearly: ForestChangeDiagnosticDataLossYearlyTwoCategory,
   tree_cover_loss_by_country_landmark_yearly: ForestChangeDiagnosticDataLossYearlyTwoCategory,
+  // Tree cover loss by country-specified classifications. For ARG, it is breakdown
+  // by OTBN categories, and for COL, it is breakdown by Frontera Agricola categories).
+  tree_cover_loss_by_country_classified_region_yearly: ForestChangeDiagnosticDataLossYearlyTwoCategory,
   /** Tree cover loss in Argentina Native Forest Land Plan (OTBN) categories */
   tree_cover_loss_arg_otbn_yearly: ForestChangeDiagnosticDataLossYearlyCategory,
   /** Tree cover loss in south east asia */
@@ -37,15 +40,21 @@ case class ForestChangeDiagnosticData(
   /** prodesLossProtectedAreasYearly */
   tree_cover_loss_prodes_wdpa_yearly: ForestChangeDiagnosticDataLossYearly,
   tree_cover_loss_prodes_primary_forest_yearly: ForestChangeDiagnosticDataLossYearly,
+  // The area in each of ARG, BRA, and COL. Only has entries for countries with
+  // non-zero overlap.
+  country_area: ForestChangeDiagnosticDataDoubleCategory,
   // The "country_specific_deforestation_*" fields use the country-specific
-  // deforestation data, which may have approximate years (because some
-  // deforestation is only specified for a range of years)
+  // deforestation data (currently only ARG and BRA), which may have approximate
+  // years (because some deforestation is only specified for a range of years)
   country_specific_deforestation_yearly: ForestChangeDiagnosticDataLossApproxYearlyCategory,
   country_specific_deforestation_wdpa_yearly: ForestChangeDiagnosticDataLossApproxYearlyTwoCategory,
   country_specific_deforestation_landmark_yearly: ForestChangeDiagnosticDataLossApproxYearlyTwoCategory,
   // This is a country-specific categorization of country-specific deforestation. For
-  // Argentina, it is a breakdown into OTBN categories.
+  // ARG, it is a breakdown by OTBN categories.
   country_specific_deforestation_classified_region_yearly: ForestChangeDiagnosticDataLossApproxYearlyTwoCategory,
+  // Breakdown by country and country-specific category (OTBN for ARG and Frontera
+  // Agricola for COL)
+  classified_region_area: ForestChangeDiagnosticDataDoubleTwoCategory,
   tree_cover_loss_brazil_biomes_yearly: ForestChangeDiagnosticDataLossYearlyCategory,
   tree_cover_extent_total: ForestChangeDiagnosticDataDouble,
   tree_cover_extent_primary_forest: ForestChangeDiagnosticDataDouble,
@@ -126,6 +135,9 @@ case class ForestChangeDiagnosticData(
       tree_cover_loss_by_country_landmark_yearly.merge(
         other.tree_cover_loss_by_country_landmark_yearly
       ),
+      tree_cover_loss_by_country_classified_region_yearly.merge(
+        other.tree_cover_loss_by_country_classified_region_yearly
+      ),
       tree_cover_loss_arg_otbn_yearly.merge(
         other.tree_cover_loss_arg_otbn_yearly
       ),
@@ -151,10 +163,12 @@ case class ForestChangeDiagnosticData(
       tree_cover_loss_prodes_primary_forest_yearly.merge(
         other.tree_cover_loss_prodes_primary_forest_yearly
       ),
+      country_area.merge(other.country_area),
       country_specific_deforestation_yearly.merge(other.country_specific_deforestation_yearly),
       country_specific_deforestation_wdpa_yearly.merge(other.country_specific_deforestation_wdpa_yearly),
       country_specific_deforestation_landmark_yearly.merge(other.country_specific_deforestation_landmark_yearly),
       country_specific_deforestation_classified_region_yearly.merge(other.country_specific_deforestation_classified_region_yearly),
+      classified_region_area.merge(other.classified_region_area),
       tree_cover_loss_brazil_biomes_yearly.merge(other.tree_cover_loss_brazil_biomes_yearly),
       tree_cover_extent_total.merge(other.tree_cover_extent_total),
       tree_cover_extent_primary_forest.merge(other.tree_cover_extent_primary_forest),
@@ -322,6 +336,7 @@ object ForestChangeDiagnosticData {
       ForestChangeDiagnosticDataLossYearlyCategory.empty,
       ForestChangeDiagnosticDataLossYearlyTwoCategory.empty,
       ForestChangeDiagnosticDataLossYearlyTwoCategory.empty,
+      ForestChangeDiagnosticDataLossYearlyTwoCategory.empty,
       ForestChangeDiagnosticDataLossYearlyCategory.empty,
       ForestChangeDiagnosticDataLossYearlyCategory.empty,
       ForestChangeDiagnosticDataLossYearlyCategory.empty,
@@ -331,10 +346,12 @@ object ForestChangeDiagnosticData {
       ForestChangeDiagnosticDataLossYearly.empty,
       ForestChangeDiagnosticDataLossYearly.empty,
       ForestChangeDiagnosticDataLossYearly.empty,
+      ForestChangeDiagnosticDataDoubleCategory.empty,
       ForestChangeDiagnosticDataLossApproxYearlyCategory.empty,
       ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.empty,
       ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.empty,
       ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.empty,
+      ForestChangeDiagnosticDataDoubleTwoCategory.empty,
       ForestChangeDiagnosticDataLossYearlyCategory.empty,
       ForestChangeDiagnosticDataDouble.empty,
       ForestChangeDiagnosticDataDouble.empty,

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleCategory.scala

@@ -19,14 +19,14 @@ case class ForestChangeDiagnosticDataDoubleCategory(
     })
   }
 
+  def round: Map[String, Double] =
+    this.value.map {
+      case (key, value) =>
+        key -> value.round
+    }
+
   def toJson: String = {
-    this.value
-      .map {
-        case (key, value) =>
-          key -> value.round
-      }
-      .asJson
-      .noSpaces
+    this.round.asJson.noSpaces
   }
 }
 

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleTwoCategory.scala

@@ -0,0 +1,62 @@
+package org.globalforestwatch.summarystats.forest_change_diagnostic
+
+import frameless.Injection
+import io.circe.syntax._
+import io.circe.parser.decode
+
+case class ForestChangeDiagnosticDataDoubleTwoCategory(
+  value: Map[String, ForestChangeDiagnosticDataDoubleCategory]
+                                                   ) extends ForestChangeDiagnosticDataParser[ForestChangeDiagnosticDataDoubleTwoCategory] {
+  def merge(
+    other: ForestChangeDiagnosticDataDoubleTwoCategory
+  ): ForestChangeDiagnosticDataDoubleTwoCategory = {
+
+    ForestChangeDiagnosticDataDoubleTwoCategory(value ++ other.value.map {
+      case (key, otherValue) =>
+        key -> value
+          .getOrElse(key, ForestChangeDiagnosticDataDoubleCategory.empty)
+          .merge(otherValue)
+    })
+  }
+
+  def toJson: String = {
+    this.value
+      .map {
+        case (key, value) =>
+          key -> value.round
+      }
+      .asJson
+      .noSpaces
+  }
+}
+
+object ForestChangeDiagnosticDataDoubleTwoCategory {
+  def empty: ForestChangeDiagnosticDataDoubleTwoCategory =
+    ForestChangeDiagnosticDataDoubleTwoCategory(Map())
+
+  def fill(
+            className1: String,
+            className2: String,
+            areaHa: Double,
+            noData: List[String] = List("", "Unknown", "Not applicable"),
+            include: Boolean = true
+          ): ForestChangeDiagnosticDataDoubleTwoCategory = {
+    if (noData.contains(className1))
+      ForestChangeDiagnosticDataDoubleTwoCategory.empty
+    else
+      ForestChangeDiagnosticDataDoubleTwoCategory(
+        Map(className1 -> ForestChangeDiagnosticDataDoubleCategory.fill(className2, areaHa, include = include))
+      )
+  }
+
+  def fromString(value: String): ForestChangeDiagnosticDataDoubleTwoCategory = {
+
+    val categories: Map[String, String] = decode[Map[String, String]](value).getOrElse(Map())
+    val newValue: Map[String, ForestChangeDiagnosticDataDoubleCategory] = categories.map { case (k, v) => (k, ForestChangeDiagnosticDataDoubleCategory.fromString(v)) }
+    ForestChangeDiagnosticDataDoubleTwoCategory(newValue)
+
+  }
+
+  implicit def injection: Injection[ForestChangeDiagnosticDataDoubleTwoCategory , String] = Injection(_.toJson, fromString)
+}
+

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossApproxYearlyCategory.scala

@@ -52,7 +52,7 @@ object ForestChangeDiagnosticDataLossApproxYearlyCategory {
             include: Boolean = true
           ): ForestChangeDiagnosticDataLossApproxYearlyCategory = {
 
-    if (noData.contains(className))
+    if (noData.contains(className) || !include)
       ForestChangeDiagnosticDataLossApproxYearlyCategory.empty
     else
       ForestChangeDiagnosticDataLossApproxYearlyCategory(

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.scala

@@ -52,7 +52,7 @@ object ForestChangeDiagnosticDataLossApproxYearlyTwoCategory {
             include: Boolean = true
           ): ForestChangeDiagnosticDataLossApproxYearlyTwoCategory = {
 
-    if (noData.contains(categoryName))
+    if (noData.contains(categoryName) || !include)
       ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.empty
     else
       ForestChangeDiagnosticDataLossApproxYearlyTwoCategory(

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyCategory.scala

@@ -48,7 +48,7 @@ object ForestChangeDiagnosticDataLossYearlyCategory {
             include: Boolean = true
           ): ForestChangeDiagnosticDataLossYearlyCategory = {
 
-    if (noData.contains(className))
+    if (noData.contains(className) || !include)
       ForestChangeDiagnosticDataLossYearlyCategory.empty
     else
       ForestChangeDiagnosticDataLossYearlyCategory(

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyTwoCategory.scala

@@ -50,7 +50,7 @@ object ForestChangeDiagnosticDataLossYearlyTwoCategory {
             include: Boolean = true
           ): ForestChangeDiagnosticDataLossYearlyTwoCategory = {
 
-    if (noData.contains(categoryName))
+    if (noData.contains(categoryName) || !include)
       ForestChangeDiagnosticDataLossYearlyTwoCategory.empty
     else
       ForestChangeDiagnosticDataLossYearlyTwoCategory(

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticGridSources.scala

@@ -29,6 +29,7 @@ case class ForestChangeDiagnosticGridSources(gridTile: GridTile, kwargs: Map[Str
   val protectedAreasByCategory: DetailedProtectedAreas = DetailedProtectedAreas(gridTile, kwargs)
   val landmark: Landmark = Landmark(gridTile, kwargs)
   val argForestLoss: ArgForestLoss = ArgForestLoss(gridTile, kwargs)
+  val colFronteraAgricola: ColFronteraAgricola = ColFronteraAgricola(gridTile, kwargs)
 
   def readWindow(
                   windowKey: SpatialKey,

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawDataGroup.scala

@@ -32,6 +32,7 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int,
                                               argPresence: Boolean,
                                               protectedAreaByCategory: String,
                                               landmarkByCategory: String,
+                                              classifiedRegion: String,
 ) {
 
   /** Produce a partial ForestChangeDiagnosticData only for the loss year in this data group */
@@ -83,7 +84,7 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int,
         protectedAreaByCategory,
         umdTreeCoverLossYear,
         totalArea,
-        include = isProtectedArea && isUMDLoss
+        include = protectedAreaByCategory != "" && isUMDLoss
       ),
     tree_cover_loss_by_country_landmark_yearly =
       ForestChangeDiagnosticDataLossYearlyTwoCategory.fill(
@@ -93,6 +94,14 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int,
         totalArea,
         include = landmarkByCategory != "" && isUMDLoss
       ),
+    tree_cover_loss_by_country_classified_region_yearly = 
+      ForestChangeDiagnosticDataLossYearlyTwoCategory.fill(
+        countryCode,
+        classifiedRegion,
+        umdTreeCoverLossYear,
+        totalArea,
+        include = classifiedRegion != "" && isUMDLoss
+    ),
     tree_cover_loss_arg_otbn_yearly = 
       ForestChangeDiagnosticDataLossYearlyCategory.fill(
       argOTBN, 
@@ -150,6 +159,8 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int,
         totalArea,
         isProdesLoss && isPrimaryForest
       ),
+    country_area = ForestChangeDiagnosticDataDoubleCategory
+      .fill(countryCode, totalArea),
     country_specific_deforestation_yearly = ForestChangeDiagnosticDataLossApproxYearlyCategory.fill(
       countryCode,
       countrySpecificLossYear,
@@ -182,6 +193,17 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int,
       } else {
         ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.empty,
       },
+      classified_region_area =
+        if (countryCode =="ARG" || countryCode == "COL") {
+          ForestChangeDiagnosticDataDoubleTwoCategory.fill(
+            countryCode,
+            classifiedRegion,
+            totalArea,
+            include = classifiedRegion != ""
+          )
+        } else {
+          ForestChangeDiagnosticDataDoubleTwoCategory.empty
+        },
     tree_cover_loss_brazil_biomes_yearly =
       ForestChangeDiagnosticDataLossYearlyCategory.fill(
         braBiomes,