Merge pull request #167 from wri/develop

Develop

.gitignore

@@ -20,6 +20,11 @@ target
 *.jar
 *.crc
 _SUCCESS
+*.bloop/
+*.bsp/
+*.metals/
+/project
+
 
 *.pyc
 .project
@@ -50,4 +55,3 @@ derby.log
 metastore_db/
 *.log
 *.tsv
-*.csv

src/main/resources/raster-catalog-default.json

@@ -10,7 +10,7 @@
     },
     {
       "name":"wdpa_protected_areas",
-      "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202204/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/geotiff/{tile_id}.tif"
+      "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202208/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/geotiff/{tile_id}.tif"
     },
     {
       "name":"gfw_oil_gas",
@@ -56,10 +56,6 @@
       "name":"gfw_forest_carbon_net_flux",
       "source_uri":"s3://gfw-data-lake/gfw_forest_carbon_net_flux/v20210331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
     },
-    {
-      "name":"wdpa_protected_areas",
-      "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202106/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif"
-    },
     {
       "name":"gfw_pixel_area",
       "source_uri":"s3://gfw-data-lake/gfw_pixel_area/v20150327/raster/epsg-4326/{grid_size}/{row_count}/m2/geotiff/{tile_id}.tif"

src/main/resources/raster-catalog-pro.json

@@ -10,11 +10,11 @@
     },
     {
       "name":"wdpa_protected_areas",
-      "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202106/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/gdal-geotiff/{tile_id}.tif"
+      "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202208/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/gdal-geotiff/{tile_id}.tif"
     },
     {
       "name":"gfw_oil_gas",
-      "source_uri":"s3://gfw-data-lake/gfw_oil_gas/v20190321/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif"
+      "source_uri":"s3://gfw-data-lake/gfw_oil_gas/v20221024/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif"
     },
     {
       "name":"idn_forest_area",
@@ -52,10 +52,6 @@
       "name":"gfw_forest_carbon_net_flux",
       "source_uri":"s3://gfw-data-lake/gfw_forest_carbon_net_flux/v20210331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/gdal-geotiff/{tile_id}.tif"
     },
-    {
-      "name":"wdpa_protected_areas",
-      "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202106/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif"
-    },
     {
       "name":"gfw_pixel_area",
       "source_uri":"s3://gfw-data-lake/gfw_pixel_area/v20150327/raster/epsg-4326/{grid_size}/{row_count}/m2/gdal-geotiff/{tile_id}.tif"

src/main/scala/org/globalforestwatch/features/SpatialFeatureDF.scala

@@ -5,6 +5,7 @@ import org.locationtech.jts
 import org.apache.spark.sql.{DataFrame, Row, SparkSession}
 import org.apache.spark.sql.functions.{col, isnull, udf}
 import org.globalforestwatch.util.GeotrellisGeometryValidator.preserveGeometryType
+import org.globalforestwatch.util.GfwGeometryFixer
 import org.locationtech.jts.geom.util.GeometryFixer
 import org.locationtech.jts.geom.{Envelope, Geometry, MultiPolygon, Polygon}
 import org.globalforestwatch.util.GfwGeometryFixer

src/main/scala/org/globalforestwatch/features/ValidatedFeatureRDD.scala

@@ -60,10 +60,10 @@ object ValidatedFeatureRDD {
   }
 
   private def splitGeometries(
-    featureType: String,
-    featureDF: DataFrame,
-    spark: SparkSession
-  ): RDD[ValidatedLocation[Geometry]] = {
+                               featureType: String,
+                               featureDF: DataFrame,
+                               spark: SparkSession
+                             ): RDD[ValidatedLocation[Geometry]] = {
     val spatialFeatureRDD: SpatialRDD[Geometry] = Adapter.toSpatialRdd(featureDF, "polyshape")
     spatialFeatureRDD.analyze()
 

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala

@@ -91,7 +91,7 @@ object ForestChangeDiagnosticAnalysis extends SummaryAnalysis {
                   } else {
                     data.copy(
                       commodity_threat_fires = fire.getOrElse(ForestChangeDiagnosticDataLossYearly.empty),
-                      tree_cover_loss_soy_yearly = data.tree_cover_loss_soy_yearly.limitToMaxYear(2020)
+                      tree_cover_loss_soy_yearly = data.tree_cover_loss_soy_yearly.limitToMaxYear(2021)
                     )
                   }
                 }
@@ -214,7 +214,7 @@ object ForestChangeDiagnosticAnalysis extends SummaryAnalysis {
       }
       .reduceByKey(_ merge _)
       .mapValues { fires =>
-        aggregateFireData(fires.merge(ForestChangeDiagnosticDataLossYearly.prefilled)).limitToMaxYear(2020)
+        aggregateFireData(fires.merge(ForestChangeDiagnosticDataLossYearly.prefilled)).limitToMaxYear(2021)
       }
   }
 

src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala

@@ -166,12 +166,12 @@ case class ForestChangeDiagnosticData(
     */
   def withUpdatedCommodityRisk(): ForestChangeDiagnosticData = {
 
-    /* Exclude the last year, limit data to 2020 to sync with palm risk tool:
+    /* Exclude the last year, limit data to 2021 to sync with palm risk tool:
     commodity_threat_deforestation, commodity_threat_peat, commodity_threat_protected_areas use year n and year n-1.
     Including information from the current year would under-represent these values as it's in progress.
     */
     val minLossYear = ForestChangeDiagnosticDataLossYearly.prefilled.value.keys.min
-    val maxLossYear = 2020
+    val maxLossYear = 2021
     val years: List[Int] = List.range(minLossYear + 1, maxLossYear + 1)
 
     val forestValueIndicator: ForestChangeDiagnosticDataValueYearly =