Merge pull request #151 from wri/develop

Merge develop to master
wri · Apr 28, 2022 · 05e757a · 05e757a
2 parents 5b9020e + ccb3f73
commit 05e757a
Show file tree

Hide file tree

Showing 50 changed files with 727 additions and 407 deletions.
diff --git a/src/main/resources/raster-catalog-default.json b/src/main/resources/raster-catalog-default.json
@@ -10,7 +10,7 @@
     },
     {
       "name":"wdpa_protected_areas",
-      "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202106/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/geotiff/{tile_id}.tif"
+      "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202204/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/geotiff/{tile_id}.tif"
     },
     {
       "name":"gfw_oil_gas",
@@ -110,7 +110,7 @@
     },
     {
       "name":"tsc_tree_cover_loss_drivers",
-      "source_uri":"s3://gfw-data-lake/tsc_tree_cover_loss_drivers/v2020/raster/epsg-4326/{grid_size}/{row_count}/driver/geotiff/{tile_id}.tif"
+      "source_uri":"s3://gfw-data-lake/tsc_tree_cover_loss_drivers/v2021.1/raster/epsg-4326/{grid_size}/{row_count}/driver/geotiff/{tile_id}.tif"
     },
     {
       "name":"umd_tree_cover_density_2010",
@@ -172,10 +172,6 @@
       "name":"gfw_belowground_carbon",
       "source_uri":"s3://gfw-data-lake/gfw_belowground_carbon/v20200824/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
     },
-    {
-      "name":"umd_tree_cover_density_2000",
-      "source_uri":"s3://gfw-data-lake/umd_tree_cover_density_2000/v1.8/raster/epsg-4326/{grid_size}/{row_count}/threshold/geotiff/{tile_id}.tif"
-    },
     {
       "name":"gfw_oil_palm",
       "source_uri":"s3://gfw-data-lake/gfw_oil_palm/v20191031/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif"
@@ -202,7 +198,7 @@
     },
     {
       "name":"umd_tree_cover_loss",
-      "source_uri":"s3://gfw-data-lake/umd_tree_cover_loss/v1.8/raster/epsg-4326/{grid_size}/{row_count}/year/geotiff/{tile_id}.tif"
+      "source_uri":"s3://gfw-data-lake/umd_tree_cover_loss/v1.9/raster/epsg-4326/{grid_size}/{row_count}/year/geotiff/{tile_id}.tif"
     },
     {
       "name":"gfw_managed_forests",
@@ -214,7 +210,7 @@
     },
     {
       "name":"gfw_mining_concessions",
-      "source_uri":"s3://gfw-data-lake/gfw_mining_concessions/v202106/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif"
+      "source_uri":"s3://gfw-data-lake/gfw_mining_concessions/v202203/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif"
     },
     {
       "name":"gfw_peatlands",
@@ -246,7 +242,7 @@
     },
     {
       "name":"gfw_planted_forests",
-      "source_uri":"s3://gfw-data-lake/gfw_planted_forests/v20201209/raster/epsg-4326/{grid_size}/{row_count}/species_simp/geotiff/{tile_id}.tif"
+      "source_uri":"s3://gfw-data-lake/gfw_planted_forests/v20201209/raster/epsg-4326/{grid_size}/{row_count}/type/geotiff/{tile_id}.tif"
     },
     {
       "name":"per_forest_concessions",
@@ -268,6 +264,14 @@
       "name":"umd_glad_landsat_alerts",
       "source_uri":"s3://gfw-data-lake/umd_glad_landsat_alerts/latest/raster/epsg-4326/{grid_size}/{row_count}/date_conf/geotiff/{tile_id}.tif"
     },
+    {
+      "name":"umd_glad_sentinel2_alerts",
+      "source_uri":"s3://gfw-data-lake/umd_glad_sentinel2_alerts/latest/raster/epsg-4326/{grid_size}/{row_count}/date_conf/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"wur_radd_alerts",
+      "source_uri":"s3://gfw-data-lake/wur_radd_alerts/latest/raster/epsg-4326/{grid_size}/{row_count}/date_conf/geotiff/{tile_id}.tif"
+    },
     {
       "name":"gfw_plantations",
       "source_uri":"s3://gfw-data-lake/gfw_plantations/v2014/raster/epsg-4326/{grid_size}/{row_count}/type/geotiff/{tile_id}.tif"
@@ -295,6 +299,54 @@
     {
       "name":"mapbox_river_basins",
       "source_uri": "s3://gfw-data-lake/mapbox_river_basins/v2018/raster/epsg-4326/{grid_size}/{row_count}/id/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_soil_carbon_stocks",
+      "source_uri": "s3://gfw-data-lake/gfw_soil_carbon_stocks/v20200724/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_age",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_age/v20220309/raster/epsg-4326/{grid_size}/{row_count}/category/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_full_extent_net_flux",
+      "source_uri": "s3://gfw-data-lake/gfw_full_extent_net_flux/v20220316/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_full_extent_aboveground_gross_removals",
+      "source_uri": "s3://gfw-data-lake/gfw_full_extent_aboveground_gross_removals/v20220309/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_full_extent_belowground_gross_removals",
+      "source_uri": "s3://gfw-data-lake/gfw_full_extent_belowground_gross_removals/v20220309/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_full_extent_co2_gross_emissions",
+      "source_uri": "s3://gfw-data-lake/gfw_full_extent_co2_gross_emissions/v20220316/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_full_extent_non_co2_gross_emissions",
+      "source_uri": "s3://gfw-data-lake/gfw_full_extent_non_co2_gross_emissions/v20220316/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"fao_ecozones",
+      "source_uri": "s3://gfw-data-lake/fao_ecozones/v2010/raster/epsg-4326/{grid_size}/{row_count}/class/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"ifl_intact_forest_landscapes_2000",
+      "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2000/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"ifl_intact_forest_landscapes_2013",
+      "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2013/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"ifl_intact_forest_landscapes_2016",
+      "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2016/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"ifl_intact_forest_landscapes_2020",
+      "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2020/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif"
     }
   ]
 }
diff --git a/src/main/resources/raster-catalog-pro.json b/src/main/resources/raster-catalog-pro.json
@@ -242,7 +242,7 @@
     },
     {
       "name":"gfw_planted_forests",
-      "source_uri":"s3://gfw-data-lake/gfw_planted_forests/v20201209/raster/epsg-4326/{grid_size}/{row_count}/species_simp/gdal-geotiff/{tile_id}.tif"
+      "source_uri":"s3://gfw-data-lake/gfw_planted_forests/v20201209/raster/epsg-4326/{grid_size}/{row_count}/type/gdal-geotiff/{tile_id}.tif"
     },
     {
       "name":"per_forest_concessions",
@@ -295,6 +295,10 @@
     {
       "name":"gfwpro_forest_change_regions",
       "source_uri": "s3://gfw-data-lake/gfwpro_forest_change_regions/v20210129/raster/epsg-4326/{grid_size}/{row_count}/bit_encoding/gdal-geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"ifl_intact_forest_landscapes_2000",
+      "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2000/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif"
     }
   ]
 }
diff --git a/src/main/scala/org/globalforestwatch/features/FeatureRDD.scala b/src/main/scala/org/globalforestwatch/features/FeatureRDD.scala
@@ -87,7 +87,8 @@ object FeatureRDD {
              feature2Type: String,
              feature2Uris: NonEmptyList[String],
              feature2Delimiter: String,
-             filters: FeatureFilter,
+             feature1Filters: FeatureFilter,
+             feature2Filters: FeatureFilter,
              spark: SparkSession
            ): RDD[geotrellis.vector.Feature[Geometry, FeatureId]] = {
 
@@ -97,7 +98,8 @@ object FeatureRDD {
       feature2Uris,
       feature2Type,
       spark,
-      filters,
+      feature1Filters,
+      feature2Filters,
       feature1Delimiter,
       feature2Delimiter,
     )

diff --git a/src/main/scala/org/globalforestwatch/features/PolygonIntersectionDF.scala b/src/main/scala/org/globalforestwatch/features/PolygonIntersectionDF.scala
@@ -16,15 +16,16 @@ object PolygonIntersectionDF {
             feature2Uris: NonEmptyList[String],
             feature2Type: String,
             spark: SparkSession,
-            filters: FeatureFilter,
+            feature1Filters: FeatureFilter,
+            feature2Filters: FeatureFilter,
             feature1Delimiter: String = "\t",
             feature2Delimiter: String = "\t"): DataFrame = {
 
     val feature1DF: DataFrame =
-      SpatialFeatureDF(feature1Uris, feature1Type, filters, "geom", spark, feature1Delimiter)
+      SpatialFeatureDF(feature1Uris, feature1Type, feature1Filters, "geom", spark, feature1Delimiter)
 
     val feature2DF: DataFrame =
-      SpatialFeatureDF(feature2Uris, feature2Type, filters, "geom", spark, feature2Delimiter)
+      SpatialFeatureDF(feature2Uris, feature2Type, feature2Filters, "geom", spark, feature2Delimiter)
 
     PolygonIntersectionDF(feature1DF, feature2DF, spark)
   }

diff --git a/src/main/scala/org/globalforestwatch/layers/FaoEcozones.scala b/src/main/scala/org/globalforestwatch/layers/FaoEcozones.scala
@@ -0,0 +1,36 @@
+package org.globalforestwatch.layers
+
+import org.globalforestwatch.grids.GridTile
+
+case class FaoEcozones(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any])
+  extends StringLayer
+    with OptionalILayer {
+
+  val datasetName = "fao_ecozones"
+  val uri: String = uriForGrid(gridTile)
+
+  def lookup(value: Int): String = value match {
+    case 1 => "Boreal coniferous forest"
+    case 2 => "Boreal mountain system"
+    case 3 => "Boreal tundra woodland"
+    case 4 => "No data"
+    case 5 => "Polar"
+    case 6 => "Subtropical desert"
+    case 7 => "Subtropical dry forest"
+    case 8 => "Subtropical humid forest"
+    case 9 => "Subtropical mountain system"
+    case 10 => "Subtropical steppe"
+    case 11 => "Temperate continental forest"
+    case 12 => "Temperate desert"
+    case 13 => "Temperate mountain system"
+    case 14 => "Temperate oceanic forest"
+    case 15 => "Temperate steppe"
+    case 16 => "Tropical desert"
+    case 17 => "Tropical dry forest"
+    case 18 => "Tropical moist deciduous forest"
+    case 19 => "Tropical mountain system"
+    case 20 => "Tropical rainforest"
+    case 21 => "Tropical shrubland"
+    case 22 => "Water"
+  }
+}
diff --git a/src/main/scala/org/globalforestwatch/layers/ForestAgeCategory.scala b/src/main/scala/org/globalforestwatch/layers/ForestAgeCategory.scala
@@ -6,13 +6,8 @@ case class ForestAgeCategory(gridTile: GridTile, model: String = "standard", kwa
   extends StringLayer
     with OptionalILayer {
 
-  val datasetName = "Na"
-
-
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
-  val uri: String =
-  //    s"$basePath/gfw_forest_age_category/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/category/gdal-geotiff/${gridTile.tileId}.tif"
-    s"s3://gfw-files/flux_1_2_1/forest_age_category/$model_suffix/${gridTile.tileId}.tif"
+  val datasetName = "gfw_forest_age"
+  val uri: String = uriForGrid(gridTile)
 
   override val externalNoDataValue = "Not applicable"
 

diff --git a/src/main/scala/org/globalforestwatch/layers/GrossCumulAbovegroundRemovalsCo2.scala b/src/main/scala/org/globalforestwatch/layers/GrossCumulAbovegroundRemovalsCo2.scala
@@ -2,18 +2,12 @@ package org.globalforestwatch.layers
 
 import org.globalforestwatch.grids.GridTile
 
-case class GrossCumulAbovegroundRemovalsCo2(gridTile: GridTile,
-                                            model: String = "standard", kwargs: Map[String, Any])
+case class GrossCumulAbovegroundRemovalsCo2(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any])
   extends FloatLayer
     with OptionalFLayer {
 
-  val datasetName = "Na"
+  val datasetName = "gfw_full_extent_aboveground_gross_removals"
 
-
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
   val uri: String =
-  //    s"$basePath/gfw_gross_cumul_aboveground_removals_co2$model_suffix/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/Mg_ha-1/geotiff/${gridTile.tileId}.tif"
-    s"s3://gfw-files/flux_1_2_1/gross_removals_AGCO2_all_forest_types/$model_suffix/${gridTile.tileId}.tif"
-  //    s"s3://gfw-files/flux_1_2_0/gross_removals_AGCO2_all_forest_types/$model_suffix/${gridTile.tileId}.tif"
-
+    uriForGrid(gridTile)
 }
diff --git a/src/main/scala/org/globalforestwatch/layers/GrossCumulBelowgroundRemovalsCo2.scala b/src/main/scala/org/globalforestwatch/layers/GrossCumulBelowgroundRemovalsCo2.scala
@@ -2,17 +2,12 @@ package org.globalforestwatch.layers
 
 import org.globalforestwatch.grids.GridTile
 
-case class GrossCumulBelowgroundRemovalsCo2(gridTile: GridTile,
-                                            model: String = "standard", kwargs: Map[String, Any])
+case class GrossCumulBelowgroundRemovalsCo2(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any])
   extends FloatLayer
     with OptionalFLayer {
 
-  val datasetName = "Na"
+  val datasetName = "gfw_full_extent_belowground_gross_removals"
 
-
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
   val uri: String =
-  //    s"$basePath/gfw_gross_cumul_belowground_removals_co2$model_suffix/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}//Mg_ha-1/geotiff/${gridTile.tileId}.tif"
-    s"s3://gfw-files/flux_1_2_1/gross_removals_BGCO2_all_forest_types/$model_suffix/${gridTile.tileId}.tif"
-  //    s"s3://gfw-files/flux_1_2_0/gross_removals_BGCO2_all_forest_types/$model_suffix/${gridTile.tileId}.tif"
+    uriForGrid(gridTile)
 }
diff --git a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala
@@ -7,12 +7,8 @@ case class GrossEmissionsCo2OnlyCo2e(gridTile: GridTile,
   extends FloatLayer
     with OptionalFLayer {
 
-  val datasetName = "Na"
+  val datasetName = "gfw_full_extent_co2_gross_emissions"
 
-
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
   val uri: String =
-  //    s"$basePath/gfw_gross_emissions_co2e_co2_only$model_suffix/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/Mg/gdal-geotiff/${gridTile.tileId}.tif"
-    s"s3://gfw-files/flux_1_2_1/gross_emissions_co2_only_co2e/$model_suffix/${gridTile.tileId}.tif"
-  //    s"s3://gfw-files/flux_1_2_0/gross_emissions_co2_only_co2e/$model_suffix/${gridTile.tileId}.tif"
+    s"s3://gfw-files/flux_1_2_2/gross_emissions_co2_only_co2e/standard/${gridTile.tileId}.tif"
 }
diff --git a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2e.scala b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2e.scala
@@ -6,12 +6,8 @@ case class GrossEmissionsNonCo2Co2e(gridTile: GridTile, model: String = "standar
   extends FloatLayer
     with OptionalFLayer {
 
-  val datasetName = "Na"
+  val datasetName = "gfw_full_extent_non_co2_gross_emissions"
 
-
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
   val uri: String =
-  //    s"$basePath/gfw_gross_emissions_co2e_non_co2$model_suffix/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/Mg/gdal-geotiff/${gridTile.tileId}.tif"
-    s"s3://gfw-files/flux_1_2_1/gross_emissions_non_co2_co2e/$model_suffix/${gridTile.tileId}.tif"
-  //    s"s3://gfw-files/flux_1_2_0/gross_emissions_non_co2_co2e/$model_suffix/${gridTile.tileId}.tif"
+    uriForGrid(gridTile)
 }
diff --git a/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala b/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala
@@ -20,47 +20,32 @@ case class IntactForestLandscapes(gridTile: GridTile, kwargs: Map[String, Any])
 
 case class IntactForestLandscapes2000(gridTile: GridTile, kwargs: Map[String, Any])
   extends BooleanLayer
-    with OptionalILayer {
-  val datasetName = "ifl_intact_forest_landscapes"
+    with OptionalILayer{
+  val datasetName = "ifl_intact_forest_landscapes_2000"
   val uri: String =
     uriForGrid(gridTile)
-
-  override def lookup(value: Int): Boolean = {
-    value match {
-      case 0 => false
-      case _ => true
-    }
-  }
 }
 
 case class IntactForestLandscapes2013(gridTile: GridTile, kwargs: Map[String, Any])
   extends BooleanLayer
-    with OptionalILayer {
-  val datasetName = "ifl_intact_forest_landscapes"
+    with OptionalILayer{
+  val datasetName = "ifl_intact_forest_landscapes_2013"
   val uri: String =
     uriForGrid(gridTile)
-
-  override def lookup(value: Int): Boolean = {
-    value match {
-      case 2016 => true
-      case 2013 => true
-      case _ => false
-    }
-  }
 }
 
 case class IntactForestLandscapes2016(gridTile: GridTile, kwargs: Map[String, Any])
   extends BooleanLayer
-    with OptionalILayer {
-  val datasetName = "ifl_intact_forest_landscapes"
+    with OptionalILayer{
+  val datasetName = "ifl_intact_forest_landscapes_2016"
   val uri: String =
     uriForGrid(gridTile)
+}
 
-  override def lookup(value: Int): Boolean = {
-    value match {
-      case 2016 => true
-      case _ => false
-    }
-  }
-
+case class IntactForestLandscapes2020(gridTile: GridTile, kwargs: Map[String, Any])
+  extends BooleanLayer
+    with OptionalILayer{
+  val datasetName = "ifl_intact_forest_landscapes_2020"
+  val uri: String =
+    uriForGrid(gridTile)
 }
diff --git a/src/main/scala/org/globalforestwatch/layers/NetFluxCo2e.scala b/src/main/scala/org/globalforestwatch/layers/NetFluxCo2e.scala
@@ -5,13 +5,8 @@ import org.globalforestwatch.grids.GridTile
 case class NetFluxCo2e(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any])
   extends FloatLayer
     with OptionalFLayer {
+  val datasetName = "gfw_full_extent_net_flux"
 
-  val datasetName = "Na"
-
-
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
   val uri: String =
-  //    s"$basePath/gfw_net_flux_co2e$model_suffix/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/Mg/gdal-geotiff/${gridTile.tileId}.tif"
-    s"s3://gfw-files/flux_1_2_1/net_flux_all_forest_types_all_drivers/$model_suffix/${gridTile.tileId}.tif"
-  //    s"s3://gfw-files/flux_1_2_0/net_flux_all_forest_types_all_drivers/$model_suffix/${gridTile.tileId}.tif"
+    uriForGrid(gridTile)
 }