Merge pull request #193 from wri/revert-prodes-split

GPTV-2215 Revert pull #175, the PRODES split by Brazilian biome

src/main/resources/raster-catalog-pro.json

@@ -197,12 +197,8 @@
       "source_uri":"s3://gfw-data-lake/gfw_managed_forests/v202106/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif"
     },
     {
-      "name":"inpe_amazon_prodes",
-      "source_uri":"s3://gfw-data-lake/inpe_amazon_prodes/v2021/raster/epsg-4326/{grid_size}/{row_count}/year/geotiff/{tile_id}.tif"
-    },
-    {
-      "name":"inpe_cerrado_prodes",
-      "source_uri":"s3://gfw-data-lake/inpe_cerrado_prodes/v2021/raster/epsg-4326/{grid_size}/{row_count}/year/geotiff/{tile_id}.tif"
+      "name":"inpe_prodes",
+      "source_uri":"s3://gfw-data-lake/inpe_prodes/v202107/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif"
     },
     {
       "name":"gfw_mining_concessions",
@@ -276,6 +272,10 @@
       "name":"umd_soy_planted_area",
       "source_uri": "s3://gfw-data-lake/umd_soy_planted_area/v2/raster/epsg-4326/{grid_size}/{row_count}/is__year_2021/gdal-geotiff/{tile_id}.tif"
     },
+    {
+      "name":"inpe_prodes",
+      "source_uri": "s3://gfw-data-lake/inpe_prodes/v202107/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif"
+    },
     {
       "name":"wwf_eco_regions",
       "source_uri": "s3://gfw-data-lake/wwf_eco_regions/v2012/raster/epsg-4326/{grid_size}/{row_count}/name/gdal-geotiff/{tile_id}.tif"

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDF.scala

@@ -29,13 +29,12 @@ object ForestChangeDiagnosticDF extends SummaryDF {
         throw new IllegalArgumentException(s"Can't produce DataFrame for $id")
     }
 
-    dataRDD
-      .map {
-        case Valid(Location(fid, data)) =>
-          (rowId(fid), RowError.empty, data)
-        case Invalid(Location(fid, err)) =>
-          (rowId(fid), RowError.fromJobError(err), ForestChangeDiagnosticData.empty)
-      }
+    dataRDD.map {
+      case Valid(Location(fid, data)) =>
+        (rowId(fid), RowError.empty, data)
+      case Invalid(Location(fid, err)) =>
+        (rowId(fid), RowError.fromJobError(err), ForestChangeDiagnosticData.empty)
+    }
       .toDF("id", "error", "data")
       .select($"id.*" :: $"error.*" :: fieldsFromCol($"data", featureFields): _*)
   }
@@ -53,27 +52,25 @@ object ForestChangeDiagnosticDF extends SummaryDF {
         throw new IllegalArgumentException("Not a CombinedFeatureId")
     }
 
-    dataRDD
-      .map {
-        case Valid(Location(fid, data)) =>
-          (rowId(fid), RowError.empty, data)
-        case Invalid(Location(fid, err)) =>
-          (rowId(fid), RowError.fromJobError(err), ForestChangeDiagnosticData.empty)
-      }
+    dataRDD.map {
+      case Valid(Location(fid, data)) =>
+        (rowId(fid), RowError.empty, data)
+      case Invalid(Location(fid, err)) =>
+        (rowId(fid), RowError.fromJobError(err), ForestChangeDiagnosticData.empty)
+    }
       .toDF("id", "error", "data")
       .select($"id.*" :: $"error.*" :: fieldsFromCol($"data", featureFields) ::: fieldsFromCol($"data", gridFields): _*)
   }
 
   def readIntermidateRDD(
     sources: NonEmptyList[String],
-    spark: SparkSession
+    spark: SparkSession,
   ): RDD[ValidatedLocation[ForestChangeDiagnosticData]] = {
     val df = FeatureDF(sources, GfwProFeature, FeatureFilter.empty, spark)
     val ds = df.select(
       colsFor[RowGridId].as[RowGridId],
       colsFor[RowError].as[RowError],
-      colsFor[ForestChangeDiagnosticData].as[ForestChangeDiagnosticData]
-    )
+      colsFor[ForestChangeDiagnosticData].as[ForestChangeDiagnosticData])
 
     ds.rdd.map { case (id, error, data) =>
       if (error.status_code == 2) Valid(Location(id.toFeatureID, data))
@@ -104,12 +101,9 @@ object ForestChangeDiagnosticDF extends SummaryDF {
     "tree_cover_loss_soy_yearly", // treeCoverLossSoyPlanedAreasYearly
     "tree_cover_loss_idn_legal_yearly", // treeCoverLossIDNForestAreaYearly
     "tree_cover_loss_idn_forest_moratorium_yearly", // treeCoverLossIDNForestMoratoriumYearly
-    "tree_cover_loss_prodes_amazon_yearly", // prodesLossAmazonYearly
-    "tree_cover_loss_prodes_cerrado_yearly", // prodesLossCerradoYearly
-    "tree_cover_loss_prodes_amazon_wdpa_yearly", // prodesLossAmazonProtectedAreasYearly
-    "tree_cover_loss_prodes_cerrado_wdpa_yearly", // prodesLossCerradoProtectedAreasYearly
-    "tree_cover_loss_prodes_amazon_primary_forest_yearly", // prodesLossProdesAmazonPrimaryForestYearly
-    "tree_cover_loss_prodes_cerrado_primary_forest_yearly", // prodesLossProdesCerradoPrimaryForestYearly
+    "tree_cover_loss_prodes_yearly", // prodesLossYearly
+    "tree_cover_loss_prodes_wdpa_yearly", // prodesLossProtectedAreasYearly
+    "tree_cover_loss_prodes_primary_forest_yearly", // prodesLossProdesPrimaryForestYearly
     "tree_cover_loss_brazil_biomes_yearly", // treeCoverLossBRABiomesYearly
     "tree_cover_extent_total", // treeCoverExtent
     "tree_cover_extent_primary_forest", // treeCoverExtentPrimaryForest
@@ -155,4 +149,4 @@ object ForestChangeDiagnosticDF extends SummaryDF {
     "plantation_in_protected_areas_area" //plantationInProtectedAreasArea
   )
 
-}
+}

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala

@@ -27,13 +27,10 @@ case class ForestChangeDiagnosticData(
   /** treeCoverLossIDNForestAreaYearly */
   tree_cover_loss_idn_legal_yearly: ForestChangeDiagnosticDataLossYearlyCategory,
   tree_cover_loss_idn_forest_moratorium_yearly: ForestChangeDiagnosticDataLossYearly,
-  tree_cover_loss_prodes_amazon_yearly: ForestChangeDiagnosticDataLossYearly,
-  tree_cover_loss_prodes_cerrado_yearly: ForestChangeDiagnosticDataLossYearly,
+  tree_cover_loss_prodes_yearly: ForestChangeDiagnosticDataLossYearly,
   /** prodesLossProtectedAreasYearly */
-  tree_cover_loss_prodes_amazon_wdpa_yearly: ForestChangeDiagnosticDataLossYearly,
-  tree_cover_loss_prodes_cerrado_wdpa_yearly: ForestChangeDiagnosticDataLossYearly,
-  tree_cover_loss_prodes_amazon_primary_forest_yearly: ForestChangeDiagnosticDataLossYearly,
-  tree_cover_loss_prodes_cerrado_primary_forest_yearly: ForestChangeDiagnosticDataLossYearly,
+  tree_cover_loss_prodes_wdpa_yearly: ForestChangeDiagnosticDataLossYearly,
+  tree_cover_loss_prodes_primary_forest_yearly: ForestChangeDiagnosticDataLossYearly,
   tree_cover_loss_brazil_biomes_yearly: ForestChangeDiagnosticDataLossYearlyCategory,
   tree_cover_extent_total: ForestChangeDiagnosticDataDouble,
   tree_cover_extent_primary_forest: ForestChangeDiagnosticDataDouble,
@@ -119,19 +116,12 @@ case class ForestChangeDiagnosticData(
       tree_cover_loss_idn_forest_moratorium_yearly.merge(
         other.tree_cover_loss_idn_forest_moratorium_yearly
       ),
-      tree_cover_loss_prodes_amazon_yearly.merge(other.tree_cover_loss_prodes_amazon_yearly),
-      tree_cover_loss_prodes_cerrado_yearly.merge(other.tree_cover_loss_prodes_cerrado_yearly),
-      tree_cover_loss_prodes_amazon_wdpa_yearly.merge(
-        other.tree_cover_loss_prodes_amazon_wdpa_yearly
+      tree_cover_loss_prodes_yearly.merge(other.tree_cover_loss_prodes_yearly),
+      tree_cover_loss_prodes_wdpa_yearly.merge(
+        other.tree_cover_loss_prodes_wdpa_yearly
       ),
-      tree_cover_loss_prodes_cerrado_wdpa_yearly.merge(
-        other.tree_cover_loss_prodes_cerrado_wdpa_yearly
-      ),
-      tree_cover_loss_prodes_amazon_primary_forest_yearly.merge(
-        other.tree_cover_loss_prodes_amazon_primary_forest_yearly
-      ),
-      tree_cover_loss_prodes_cerrado_primary_forest_yearly.merge(
-        other.tree_cover_loss_prodes_cerrado_primary_forest_yearly
+      tree_cover_loss_prodes_primary_forest_yearly.merge(
+        other.tree_cover_loss_prodes_primary_forest_yearly
       ),
       tree_cover_loss_brazil_biomes_yearly.merge(other.tree_cover_loss_brazil_biomes_yearly),
       tree_cover_extent_total.merge(other.tree_cover_extent_total),
@@ -181,27 +171,25 @@ case class ForestChangeDiagnosticData(
     )
   }
 
-  /** @see
-    *   https://docs.google.com/presentation/d/1nAq4mFNkv1q5vFvvXWReuLr4Znvr-1q-BDi6pl_5zTU/edit#slide=id.p
+  /**
+    * @see https://docs.google.com/presentation/d/1nAq4mFNkv1q5vFvvXWReuLr4Znvr-1q-BDi6pl_5zTU/edit#slide=id.p
     */
   def withUpdatedCommodityRisk(): ForestChangeDiagnosticData = {
 
     /* Exclude the last year, limit data to 2021 to sync with palm risk tool:
     commodity_threat_deforestation, commodity_threat_peat, commodity_threat_protected_areas use year n and year n-1.
     Including information from the current year would under-represent these values as it's in progress.
-     */
+    */
     val minLossYear = ForestChangeDiagnosticDataLossYearly.prefilled.value.keys.min
     val maxLossYear = 2021
     val years: List[Int] = List.range(minLossYear + 1, maxLossYear + 1)
 
     val forestValueIndicator: ForestChangeDiagnosticDataValueYearly =
-      ForestChangeDiagnosticDataValueYearly
-        .fill(
-          filtered_tree_cover_extent.value,
-          filtered_tree_cover_loss_yearly.value,
-          2
-        )
-        .limitToMaxYear(maxLossYear)
+      ForestChangeDiagnosticDataValueYearly.fill(
+        filtered_tree_cover_extent.value,
+        filtered_tree_cover_loss_yearly.value,
+        2
+      ).limitToMaxYear(maxLossYear)
 
     val peatValueIndicator: ForestChangeDiagnosticDataValueYearly =
       ForestChangeDiagnosticDataValueYearly.fill(peat_area.value).limitToMaxYear(maxLossYear)
@@ -212,69 +200,66 @@ case class ForestChangeDiagnosticData(
     val deforestationThreatIndicator: ForestChangeDiagnosticDataLossYearly =
       ForestChangeDiagnosticDataLossYearly(
         SortedMap(
-          years.map(year =>
-            (
-              year, {
+          years.map(
+            year =>
+              (year, {
                 // Somehow the compiler cannot infer the types correctly
                 // I hence declare them here explicitly to help him out.
                 val thisYearLoss: Double =
-                  filtered_tree_cover_loss_yearly.value
-                    .getOrElse(year, 0)
+                filtered_tree_cover_loss_yearly.value
+                  .getOrElse(year, 0)
 
                 val lastYearLoss: Double =
                   filtered_tree_cover_loss_yearly.value
                     .getOrElse(year - 1, 0)
 
                 thisYearLoss + lastYearLoss
-              }
-            )
+              })
           ): _*
         )
       ).limitToMaxYear(maxLossYear)
 
     val peatThreatIndicator: ForestChangeDiagnosticDataLossYearly =
       ForestChangeDiagnosticDataLossYearly(
         SortedMap(
-          years.map(year =>
-            (
-              year, {
+          years.map(
+            year =>
+              (year, {
                 // Somehow the compiler cannot infer the types correctly
                 // I hence declare them here explicitly to help him out.
                 val thisYearPeatLoss: Double =
-                  filtered_tree_cover_loss_peat_yearly.value
-                    .getOrElse(year, 0)
+                filtered_tree_cover_loss_peat_yearly.value
+                  .getOrElse(year, 0)
 
                 val lastYearPeatLoss: Double =
                   filtered_tree_cover_loss_peat_yearly.value
                     .getOrElse(year - 1, 0)
 
                 thisYearPeatLoss + lastYearPeatLoss + plantation_on_peat_area.value
 
-              }
-            )
+              })
           ): _*
         )
       ).limitToMaxYear(maxLossYear)
 
     val protectedAreaThreatIndicator: ForestChangeDiagnosticDataLossYearly =
       ForestChangeDiagnosticDataLossYearly(
         SortedMap(
-          years.map(year =>
-            (
-              year, {
+          years.map(
+            year =>
+              (year, {
                 // Somehow the compiler cannot infer the types correctly
                 // I hence declare them here explicitly to help him out.
                 val thisYearProtectedAreaLoss: Double =
-                  filtered_tree_cover_loss_protected_areas_yearly.value
-                    .getOrElse(year, 0)
+                filtered_tree_cover_loss_protected_areas_yearly.value
+                  .getOrElse(year, 0)
 
                 val lastYearProtectedAreaLoss: Double =
                   filtered_tree_cover_loss_protected_areas_yearly.value
                     .getOrElse(year - 1, 0)
 
                 thisYearProtectedAreaLoss + lastYearProtectedAreaLoss + plantation_in_protected_areas_area.value
-              }
-            )
+              })
           ): _*
         )
       ).limitToMaxYear(maxLossYear)
@@ -285,8 +270,7 @@ case class ForestChangeDiagnosticData(
       commodity_value_protected_areas = protectedAreaValueIndicator,
       commodity_threat_deforestation = deforestationThreatIndicator,
       commodity_threat_peat = peatThreatIndicator,
-      commodity_threat_protected_areas = protectedAreaThreatIndicator
-    )
+      commodity_threat_protected_areas = protectedAreaThreatIndicator)
   }
 
 }
@@ -310,9 +294,6 @@ object ForestChangeDiagnosticData {
       ForestChangeDiagnosticDataLossYearly.empty,
       ForestChangeDiagnosticDataLossYearly.empty,
       ForestChangeDiagnosticDataLossYearly.empty,
-      ForestChangeDiagnosticDataLossYearly.empty,
-      ForestChangeDiagnosticDataLossYearly.empty,
-      ForestChangeDiagnosticDataLossYearly.empty,
       ForestChangeDiagnosticDataLossYearlyCategory.empty,
       ForestChangeDiagnosticDataDouble.empty,
       ForestChangeDiagnosticDataDouble.empty,
@@ -351,17 +332,17 @@ object ForestChangeDiagnosticData {
       ForestChangeDiagnosticDataLossYearly.empty,
       ForestChangeDiagnosticDataLossYearly.empty,
       ForestChangeDiagnosticDataLossYearly.empty,
-      ForestChangeDiagnosticDataLossYearly.empty
+      ForestChangeDiagnosticDataLossYearly.empty,
     )
 
   implicit val lossDataSemigroup: Semigroup[ForestChangeDiagnosticData] =
     new Semigroup[ForestChangeDiagnosticData] {
-      def combine(x: ForestChangeDiagnosticData, y: ForestChangeDiagnosticData): ForestChangeDiagnosticData =
+      def combine(x: ForestChangeDiagnosticData,
+                  y: ForestChangeDiagnosticData): ForestChangeDiagnosticData =
         x.merge(y)
     }
 
   implicit def dataExpressionEncoder: ExpressionEncoder[ForestChangeDiagnosticData] =
-    frameless
-      .TypedExpressionEncoder[ForestChangeDiagnosticData]
+    frameless.TypedExpressionEncoder[ForestChangeDiagnosticData]
       .asInstanceOf[ExpressionEncoder[ForestChangeDiagnosticData]]
 }