Merge pull request #206 from wri/feature/flux_model_v1_3_1_run

Feature/flux model v1 3 1 run

build.sbt

@@ -260,16 +260,24 @@ sparkEmrConfigs := List(
 
     //    "spark.kryoserializer.buffer.max" -> "2047m",
 
-    //     Best practice 4: Always set up a garbage collector when handling large volume of data through Spark.
+    // Best practice 4: Always set up a garbage collector when handling large volume of data through Spark.
     // Use these GC strategy to avoid java.lang.OutOfMemoryError: GC overhead limit exceeded
      "spark.executor.defaultJavaOptions" -> "-XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:OnOutOfMemoryError='kill -9 %p'",
      "spark.driver.defaultJavaOptions" -> "-XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:OnOutOfMemoryError='kill -9 %p'",
 
-    // set this environment variable for GDAL to use request payer method for S3 files
-    "spark.yarn.appMasterEnv.AWS_REQUEST_PAYER" -> "requester",
+    // set this environment variable for GDAL to use request payer method for S3 files.
+    // Added some of these to resolve 503 errors using 200 worker instances.
     "spark.executorEnv.AWS_REQUEST_PAYER" -> "requester",
+    "spark.yarn.appMasterEnv.AWS_REQUEST_PAYER" -> "requester",
+    "spark.executorEnv.GDAL_HTTP_MAX_RETRY" -> "3",
+    "spark.yarn.appMasterEnv.GDAL_HTTP_MAX_RETRY" -> "3",
+    "spark.executorEnv.GDAL_DISABLE_READDIR_ON_OPEN" -> "EMPTY_DIR",
+    "spark.yarn.appMasterEnv.GDAL_DISABLE_READDIR_ON_OPEN" -> "EMPTY_DIR",
+    "spark.executorEnv.GDAL_HTTP_RETRY_DELAY" -> "10",
+    "spark.yarn.appMasterEnv.GDAL_HTTP_RETRY_DELAY" -> "10"
+
 
-  ),
+),
   //  EmrConfig("spark-env").withProperties(
   //    "LD_LIBRARY_PATH" -> "/usr/local/lib"
   //  ),

src/main/resources/raster-catalog-default.json

@@ -255,36 +255,111 @@
 
 
     {
-      "name":"gfw_aboveground_carbon",
-      "source_uri":"s3://gfw-data-lake/gfw_aboveground_carbon/v20230322/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
+      "name":"gfw_forest_flux_full_extent_removal_factor_aboveground",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_removal_factor_aboveground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1_yr-1/geotiff/{tile_id}.tif"
     },
     {
-      "name":"gfw_belowground_carbon",
-      "source_uri":"s3://gfw-data-lake/gfw_belowground_carbon/v20230322/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
+      "name":"gfw_forest_flux_full_extent_removal_factor_belowground",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_removal_factor_belowground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1_yr-1/geotiff/{tile_id}.tif"
     },
     {
-      "name":"gfw_soil_carbon",
-      "source_uri": "s3://gfw-data-lake/gfw_soil_carbon/v20230322.2/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
+      "name":"gfw_forest_flux_full_extent_gross_removals_aboveground",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_removals_aboveground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
     },
     {
-      "name":"gfw_full_extent_net_flux",
-      "source_uri": "s3://gfw-data-lake/gfw_full_extent_net_flux/v20230407/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+      "name":"gfw_forest_flux_full_extent_gross_removals_belowground",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_removals_belowground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
     },
     {
-      "name":"gfw_full_extent_aboveground_gross_removals",
-      "source_uri": "s3://gfw-data-lake/gfw_full_extent_aboveground_gross_removals/v20230331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+      "name":"gfw_forest_flux_full_extent_gross_emissions_co2_only_biomass_soil",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_co2_only_biomass_soil/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
     },
     {
-      "name":"gfw_full_extent_belowground_gross_removals",
-      "source_uri": "s3://gfw-data-lake/gfw_full_extent_belowground_gross_removals/v20230331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+      "name":"gfw_forest_flux_full_extent_gross_emissions_non_co2_biomass_soil",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_non_co2_biomass_soil/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
     },
     {
-      "name":"gfw_full_extent_co2_gross_emissions_biomass_soil",
-      "source_uri": "s3://gfw-data-lake/gfw_full_extent_co2_gross_emissions/v20230407/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+      "name":"gfw_forest_flux_full_extent_gross_emissions_co2_only_soil_only",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_co2_only_soil_only/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
     },
     {
-      "name":"gfw_full_extent_non_co2_gross_emissions_biomass_soil",
-      "source_uri": "s3://gfw-data-lake/gfw_full_extent_non_co2_gross_emissions/v20230407/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+      "name":"gfw_forest_flux_full_extent_gross_emissions_non_co2_soil_only",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_non_co2_soil_only/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_flux_full_extent_net_flux",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_net_flux/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
+    },
+
+
+    {
+      "name":"gfw_forest_flux_aboveground_carbon_stock_in_emissions_year",
+      "source_uri":"s3://gfw-data-lake/gfw_forest_flux_aboveground_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_flux_belowground_carbon_stock_in_emissions_year",
+      "source_uri":"s3://gfw-data-lake/gfw_forest_flux_belowground_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_flux_deadwood_carbon_stock_in_emissions_year",
+      "source_uri":"s3://gfw-data-lake/gfw_forest_flux_deadwood_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_flux_litter_carbon_stock_in_emissions_year",
+      "source_uri":"s3://gfw-data-lake/gfw_forest_flux_litter_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_flux_soil_carbon_stock_in_emissions_year",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_soil_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_aboveground_carbon_stock_2000",
+      "source_uri":"s3://gfw-data-lake/gfw_aboveground_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_belowground_carbon_stock_2000",
+      "source_uri":"s3://gfw-data-lake/gfw_belowground_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_deadwood_carbon_stock_2000",
+      "source_uri":"s3://gfw-data-lake/gfw_deadwood_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_litter_carbon_stock_2000",
+      "source_uri":"s3://gfw-data-lake/gfw_litter_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_soil_carbon_stock_2000",
+      "source_uri": "s3://gfw-data-lake/gfw_soil_carbon_stock_2000/v20231108/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
+    },
+
+    {
+      "name":"gfw_forest_flux_soil_carbon_stock_2000_stdev",
+      "source_uri":"s3://gfw-data-lake/gfw_forest_flux_soil_carbon_stock_2000_stdev/v20231108/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_flux_removal_factor_aboveground_carbon_stdev",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_removal_factor_aboveground_carbon_stdev/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1_yr-1/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_flux_model_extent",
+      "source_uri":"s3://gfw-data-lake/gfw_forest_flux_model_extent/v20231114/raster/epsg-4326/{grid_size}/{row_count}/ha/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_flux_removal_forest_type",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_removal_forest_type/v20231114/raster/epsg-4326/{grid_size}/{row_count}/source/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_flux_forest_age_category",
+      "source_uri":"s3://gfw-data-lake/gfw_forest_flux_forest_age_category/v20231114/raster/epsg-4326/{grid_size}/{row_count}/ageCategory/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_flux_gross_emissions_node_codes",
+      "source_uri": "s3://gfw-data-lake/gfw_forest_flux_gross_emissions_node_codes/v20231114/raster/epsg-4326/{grid_size}/{row_count}/category/geotiff/{tile_id}.tif"
+    },
+    {
+      "name":"gfw_forest_flux_planted_forest_type",
+      "source_uri": "s3://gfw-data-lake/gfw_planted_forests/v20230911/raster/epsg-4326/{grid_size}/{row_count}/simpleName/geotiff/{tile_id}.tif"
     }
   ]
 }

src/main/scala/org/globalforestwatch/layers/AbovegroundBiomass2000.scala

@@ -3,6 +3,7 @@ package org.globalforestwatch.layers
 import org.globalforestwatch.grids.GridTile
 
 case class AbovegroundBiomass2000(gridTile: GridTile, kwargs: Map[String, Any]) extends DoubleLayer with OptionalDLayer {
+
   val datasetName = "whrc_aboveground_biomass_density_2000"
   val uri: String = uriForGrid(gridTile, kwargs)
 }

src/main/scala/org/globalforestwatch/layers/AbovegroundCarbon2000.scala

@@ -6,16 +6,16 @@ case class AbovegroundCarbon2000(gridTile: GridTile, model: String = "standard",
   extends FloatLayer
     with OptionalFLayer {
 
-  val datasetName = "gfw_aboveground_carbon"
+  val datasetName = "gfw_aboveground_carbon_stock_2000"
 
   val uri: String =
     uriForGrid(gridTile, kwargs)
 
 
-//  // For carbonflux package run only
+//  // For carbon_sensitivity run only (but not currently functional)
 //  val datasetName = "Na"
 //
 //  val model_suffix: String = if (model == "standard") "standard" else s"$model"
 //  val uri: String =
-//    s"s3://gfw-files/flux_1_2_3/agc_2000/$model_suffix/${gridTile.tileId}.tif"
+//    s"s3://gfw-data-lake/gfw_aboveground_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
 }

src/main/scala/org/globalforestwatch/layers/AbovegroundCarbonEmisYear.scala

@@ -6,9 +6,15 @@ case class AbovegroundCarbonEmisYear(gridTile: GridTile, model: String = "standa
   extends FloatLayer
     with OptionalFLayer {
 
-  val datasetName = "Na"
+  val datasetName = "gfw_forest_flux_aboveground_carbon_stock_in_emissions_year"
 
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
   val uri: String =
-      s"s3://gfw-files/flux_1_2_3/agc_emis_year/$model_suffix/${gridTile.tileId}.tif"
+    uriForGrid(gridTile, kwargs)
+
+  //  // For carbon_sensitivity run only (but not currently functional)
+  //  val datasetName = "Na"
+
+  //  val model_suffix: String = if (model == "standard") "standard" else s"$model"
+  //  val uri: String =
+  //    s"s3://gfw-data-lake/gfw_forest_flux_aboveground_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
 }

src/main/scala/org/globalforestwatch/layers/AnnualAbovegroundRemovalFactorCarbon.scala

@@ -6,9 +6,15 @@ case class AnnualAbovegroundRemovalFactorCarbon(gridTile: GridTile, model: Strin
   extends FloatLayer
     with OptionalFLayer {
 
-  val datasetName = "Na"
+  val datasetName = "gfw_forest_flux_full_extent_removal_factor_aboveground"
 
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
   val uri: String =
-      s"s3://gfw-files/flux_1_2_3/annual_removal_factor_AGC_all_forest_types/$model_suffix/${gridTile.tileId}.tif"
+    uriForGrid(gridTile, kwargs)
+
+  //  // For carbon_sensitivity run only (but not currently functional)
+  //  val datasetName = "Na"
+  //
+  //  val model_suffix: String = if (model == "standard") "standard" else s"$model"
+  //  val uri: String =
+  //      s"s3://gfw-data-lake/gfw_forest_flux_full_extent_removal_factor_aboveground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1_yr-1/geotiff/{tile_id}.tif"
 }

src/main/scala/org/globalforestwatch/layers/AnnualBelowgroundRemovalFactorCarbon.scala

@@ -6,9 +6,14 @@ case class AnnualBelowgroundRemovalFactorCarbon(gridTile: GridTile, model: Strin
   extends FloatLayer
     with OptionalFLayer {
 
-  val datasetName = "Na"
+  val datasetName = "gfw_forest_flux_full_extent_removal_factor_belowground"
 
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
   val uri: String =
-      s"s3://gfw-files/flux_1_2_3/annual_removal_factor_BGC_all_forest_types/$model_suffix/${gridTile.tileId}.tif"
+    uriForGrid(gridTile, kwargs)
+
+  //  // For carbon_sensitivity run only (but not currently functional)
+  //  val datasetName = "Na"
+  //  val model_suffix: String = if (model == "standard") "standard" else s"$model"
+  //  val uri: String =
+  //      s"s3://gfw-data-lake/gfw_forest_flux_full_extent_removal_factor_belowground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1_yr-1/geotiff/{tile_id}.tif"
 }

src/main/scala/org/globalforestwatch/layers/BelowgroundCarbon2000.scala

@@ -6,16 +6,16 @@ case class BelowgroundCarbon2000(gridTile: GridTile, model: String = "standard",
   extends FloatLayer
     with OptionalFLayer {
 
-  val datasetName = "gfw_belowground_carbon"
+  val datasetName = "gfw_belowground_carbon_stock_2000"
 
   val uri: String =
     uriForGrid(gridTile, kwargs)
 
 
-//  // For carbonflux package run only
+//  // For carbon_sensitivity run only (but not currently functional)
 //  val datasetName = "Na"
 //
 //  val model_suffix: String = if (model == "standard") "standard" else s"$model"
 //  val uri: String =
-//    s"s3://gfw-files/flux_1_2_3/bgc_2000/$model_suffix/${gridTile.tileId}.tif"
+//    s"s3://gfw-data-lake/gfw_belowground_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
 }

src/main/scala/org/globalforestwatch/layers/BelowgroundCarbonEmisYear.scala

@@ -6,9 +6,14 @@ case class BelowgroundCarbonEmisYear(gridTile: GridTile, model: String = "standa
   extends FloatLayer
     with OptionalFLayer {
 
-  val datasetName = "Na"
+  val datasetName = "gfw_forest_flux_belowground_carbon_stock_in_emissions_year"
 
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
   val uri: String =
-      s"s3://gfw-files/flux_1_2_3/bgc_emis_year/$model_suffix/${gridTile.tileId}.tif"
+    uriForGrid(gridTile, kwargs)
+
+  //  // For carbon_sensitivity run only (but not currently functional)
+  //  val datasetName = "Na"
+  //  val model_suffix: String = if (model == "standard") "standard" else s"$model"
+  //  val uri: String =
+  //      s"s3://gfw-data-lake/gfw_forest_flux_belowground_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
 }

src/main/scala/org/globalforestwatch/layers/DeadwoodCarbon2000.scala

@@ -6,16 +6,16 @@ case class DeadwoodCarbon2000(gridTile: GridTile, model: String = "standard", kw
   extends FloatLayer
     with OptionalFLayer {
 
-//  val datasetName = "gfw_deadwood_carbon"
-//
-//  val uri: String =
-//    uriForGrid(gridTile, kwargs)
+  val datasetName = "gfw_deadwood_carbon_stock_2000"
 
+  val uri: String =
+    uriForGrid(gridTile, kwargs)
 
-  // For carbonflux package run only
-  val datasetName = "Na"
 
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
-  val uri: String =
-    s"s3://gfw-files/flux_1_2_3/deadwood_carbon_2000/$model_suffix/${gridTile.tileId}.tif"
+//  // For carbon_sensitivity run only (but not currently functional)
+//  val datasetName = "Na"
+//
+//  val model_suffix: String = if (model == "standard") "standard" else s"$model"
+//  val uri: String =
+//    s"s3://gfw-data-lake/gfw_deadwood_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
 }

src/main/scala/org/globalforestwatch/layers/DeadwoodCarbonEmisYear.scala

@@ -6,9 +6,14 @@ case class DeadwoodCarbonEmisYear(gridTile: GridTile, model: String = "standard"
   extends FloatLayer
     with OptionalFLayer {
 
-  val datasetName = "Na"
+  val datasetName = "gfw_forest_flux_deadwood_carbon_stock_in_emissions_year"
 
-  val model_suffix: String = if (model == "standard") "standard" else s"$model"
   val uri: String =
-      s"s3://gfw-files/flux_1_2_3/deadwood_emis_year/$model_suffix/${gridTile.tileId}.tif"
+    uriForGrid(gridTile, kwargs)
+
+  //  // For carbon_sensitivity run only (but not currently functional)
+  //  val datasetName = "Na"
+  //  val model_suffix: String = if (model == "standard") "standard" else s"$model"
+  //  val uri: String =
+  //      s"s3://gfw-data-lake/gfw_forest_flux_deadwood_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif"
 }

src/main/scala/org/globalforestwatch/layers/ForestFluxModelAgeCategory.scala

@@ -0,0 +1,30 @@
+package org.globalforestwatch.layers
+
+import org.globalforestwatch.grids.GridTile
+
+case class ForestFluxModelAgeCategory(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any])
+  extends StringLayer
+    with OptionalILayer {
+
+  val datasetName = "gfw_forest_flux_forest_age_category"
+
+  val uri: String =
+    uriForGrid(gridTile, kwargs)
+
+
+  //  // For carbon_sensitivity run only (but not currently functional)
+  //  val datasetName = "Na"
+  //
+  //  val model_suffix: String = if (model == "standard") "standard" else s"$model"
+  //  val uri: String =
+  //    s"s3://gfw-data-lake/gfw_forest_flux_forest_age_category/v20231114/raster/epsg-4326/{grid_size}/{row_count}/ageCategory/geotiff/{tile_id}.tif"
+
+  override val externalNoDataValue = "Not applicable"
+
+  def lookup(value: Int): String = value match {
+    case 1 => "Secondary forest <=20 years"
+    case 2  => "Secondary forest >20 years"
+    case 3  => "Primary forest"
+    case _ => "Unknown"
+  }
+}