diff --git a/build.sbt b/build.sbt index 943ccafa..c9e33546 100644 --- a/build.sbt +++ b/build.sbt @@ -260,16 +260,24 @@ sparkEmrConfigs := List( // "spark.kryoserializer.buffer.max" -> "2047m", - // Best practice 4: Always set up a garbage collector when handling large volume of data through Spark. + // Best practice 4: Always set up a garbage collector when handling large volume of data through Spark. // Use these GC strategy to avoid java.lang.OutOfMemoryError: GC overhead limit exceeded "spark.executor.defaultJavaOptions" -> "-XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:OnOutOfMemoryError='kill -9 %p'", "spark.driver.defaultJavaOptions" -> "-XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:OnOutOfMemoryError='kill -9 %p'", - // set this environment variable for GDAL to use request payer method for S3 files - "spark.yarn.appMasterEnv.AWS_REQUEST_PAYER" -> "requester", + // set this environment variable for GDAL to use request payer method for S3 files. + // Added some of these to resolve 503 errors using 200 worker instances. "spark.executorEnv.AWS_REQUEST_PAYER" -> "requester", + "spark.yarn.appMasterEnv.AWS_REQUEST_PAYER" -> "requester", + "spark.executorEnv.GDAL_HTTP_MAX_RETRY" -> "3", + "spark.yarn.appMasterEnv.GDAL_HTTP_MAX_RETRY" -> "3", + "spark.executorEnv.GDAL_DISABLE_READDIR_ON_OPEN" -> "EMPTY_DIR", + "spark.yarn.appMasterEnv.GDAL_DISABLE_READDIR_ON_OPEN" -> "EMPTY_DIR", + "spark.executorEnv.GDAL_HTTP_RETRY_DELAY" -> "10", + "spark.yarn.appMasterEnv.GDAL_HTTP_RETRY_DELAY" -> "10" + - ), +), // EmrConfig("spark-env").withProperties( // "LD_LIBRARY_PATH" -> "/usr/local/lib" // ), diff --git a/src/main/resources/raster-catalog-default.json b/src/main/resources/raster-catalog-default.json index b2210716..6056374a 100644 --- a/src/main/resources/raster-catalog-default.json +++ b/src/main/resources/raster-catalog-default.json @@ -255,36 +255,111 @@ { - "name":"gfw_aboveground_carbon", - "source_uri":"s3://gfw-data-lake/gfw_aboveground_carbon/v20230322/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" + "name":"gfw_forest_flux_full_extent_removal_factor_aboveground", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_removal_factor_aboveground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1_yr-1/geotiff/{tile_id}.tif" }, { - "name":"gfw_belowground_carbon", - "source_uri":"s3://gfw-data-lake/gfw_belowground_carbon/v20230322/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" + "name":"gfw_forest_flux_full_extent_removal_factor_belowground", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_removal_factor_belowground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1_yr-1/geotiff/{tile_id}.tif" }, { - "name":"gfw_soil_carbon", - "source_uri": "s3://gfw-data-lake/gfw_soil_carbon/v20230322.2/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" + "name":"gfw_forest_flux_full_extent_gross_removals_aboveground", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_removals_aboveground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" }, { - "name":"gfw_full_extent_net_flux", - "source_uri": "s3://gfw-data-lake/gfw_full_extent_net_flux/v20230407/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + "name":"gfw_forest_flux_full_extent_gross_removals_belowground", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_removals_belowground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" }, { - "name":"gfw_full_extent_aboveground_gross_removals", - "source_uri": "s3://gfw-data-lake/gfw_full_extent_aboveground_gross_removals/v20230331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + "name":"gfw_forest_flux_full_extent_gross_emissions_co2_only_biomass_soil", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_co2_only_biomass_soil/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" }, { - "name":"gfw_full_extent_belowground_gross_removals", - "source_uri": "s3://gfw-data-lake/gfw_full_extent_belowground_gross_removals/v20230331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + "name":"gfw_forest_flux_full_extent_gross_emissions_non_co2_biomass_soil", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_non_co2_biomass_soil/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" }, { - "name":"gfw_full_extent_co2_gross_emissions_biomass_soil", - "source_uri": "s3://gfw-data-lake/gfw_full_extent_co2_gross_emissions/v20230407/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + "name":"gfw_forest_flux_full_extent_gross_emissions_co2_only_soil_only", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_co2_only_soil_only/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" }, { - "name":"gfw_full_extent_non_co2_gross_emissions_biomass_soil", - "source_uri": "s3://gfw-data-lake/gfw_full_extent_non_co2_gross_emissions/v20230407/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + "name":"gfw_forest_flux_full_extent_gross_emissions_non_co2_soil_only", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_non_co2_soil_only/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_flux_full_extent_net_flux", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_full_extent_net_flux/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + }, + + + { + "name":"gfw_forest_flux_aboveground_carbon_stock_in_emissions_year", + "source_uri":"s3://gfw-data-lake/gfw_forest_flux_aboveground_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_flux_belowground_carbon_stock_in_emissions_year", + "source_uri":"s3://gfw-data-lake/gfw_forest_flux_belowground_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_flux_deadwood_carbon_stock_in_emissions_year", + "source_uri":"s3://gfw-data-lake/gfw_forest_flux_deadwood_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_flux_litter_carbon_stock_in_emissions_year", + "source_uri":"s3://gfw-data-lake/gfw_forest_flux_litter_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_flux_soil_carbon_stock_in_emissions_year", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_soil_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_aboveground_carbon_stock_2000", + "source_uri":"s3://gfw-data-lake/gfw_aboveground_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_belowground_carbon_stock_2000", + "source_uri":"s3://gfw-data-lake/gfw_belowground_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_deadwood_carbon_stock_2000", + "source_uri":"s3://gfw-data-lake/gfw_deadwood_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_litter_carbon_stock_2000", + "source_uri":"s3://gfw-data-lake/gfw_litter_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_soil_carbon_stock_2000", + "source_uri": "s3://gfw-data-lake/gfw_soil_carbon_stock_2000/v20231108/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" + }, + + { + "name":"gfw_forest_flux_soil_carbon_stock_2000_stdev", + "source_uri":"s3://gfw-data-lake/gfw_forest_flux_soil_carbon_stock_2000_stdev/v20231108/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_flux_removal_factor_aboveground_carbon_stdev", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_removal_factor_aboveground_carbon_stdev/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1_yr-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_flux_model_extent", + "source_uri":"s3://gfw-data-lake/gfw_forest_flux_model_extent/v20231114/raster/epsg-4326/{grid_size}/{row_count}/ha/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_flux_removal_forest_type", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_removal_forest_type/v20231114/raster/epsg-4326/{grid_size}/{row_count}/source/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_flux_forest_age_category", + "source_uri":"s3://gfw-data-lake/gfw_forest_flux_forest_age_category/v20231114/raster/epsg-4326/{grid_size}/{row_count}/ageCategory/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_flux_gross_emissions_node_codes", + "source_uri": "s3://gfw-data-lake/gfw_forest_flux_gross_emissions_node_codes/v20231114/raster/epsg-4326/{grid_size}/{row_count}/category/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_flux_planted_forest_type", + "source_uri": "s3://gfw-data-lake/gfw_planted_forests/v20230911/raster/epsg-4326/{grid_size}/{row_count}/simpleName/geotiff/{tile_id}.tif" } ] } \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/layers/AbovegroundBiomass2000.scala b/src/main/scala/org/globalforestwatch/layers/AbovegroundBiomass2000.scala index b657376f..c2f2d0c0 100644 --- a/src/main/scala/org/globalforestwatch/layers/AbovegroundBiomass2000.scala +++ b/src/main/scala/org/globalforestwatch/layers/AbovegroundBiomass2000.scala @@ -3,6 +3,7 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile case class AbovegroundBiomass2000(gridTile: GridTile, kwargs: Map[String, Any]) extends DoubleLayer with OptionalDLayer { + val datasetName = "whrc_aboveground_biomass_density_2000" val uri: String = uriForGrid(gridTile, kwargs) } diff --git a/src/main/scala/org/globalforestwatch/layers/AbovegroundCarbon2000.scala b/src/main/scala/org/globalforestwatch/layers/AbovegroundCarbon2000.scala index ed4196b7..789a73e2 100644 --- a/src/main/scala/org/globalforestwatch/layers/AbovegroundCarbon2000.scala +++ b/src/main/scala/org/globalforestwatch/layers/AbovegroundCarbon2000.scala @@ -6,16 +6,16 @@ case class AbovegroundCarbon2000(gridTile: GridTile, model: String = "standard", extends FloatLayer with OptionalFLayer { - val datasetName = "gfw_aboveground_carbon" + val datasetName = "gfw_aboveground_carbon_stock_2000" val uri: String = uriForGrid(gridTile, kwargs) -// // For carbonflux package run only +// // For carbon_sensitivity run only (but not currently functional) // val datasetName = "Na" // // val model_suffix: String = if (model == "standard") "standard" else s"$model" // val uri: String = -// s"s3://gfw-files/flux_1_2_3/agc_2000/$model_suffix/${gridTile.tileId}.tif" +// s"s3://gfw-data-lake/gfw_aboveground_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/AbovegroundCarbonEmisYear.scala b/src/main/scala/org/globalforestwatch/layers/AbovegroundCarbonEmisYear.scala index 916a3ef2..3ee016f2 100644 --- a/src/main/scala/org/globalforestwatch/layers/AbovegroundCarbonEmisYear.scala +++ b/src/main/scala/org/globalforestwatch/layers/AbovegroundCarbonEmisYear.scala @@ -6,9 +6,15 @@ case class AbovegroundCarbonEmisYear(gridTile: GridTile, model: String = "standa extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_forest_flux_aboveground_carbon_stock_in_emissions_year" - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - s"s3://gfw-files/flux_1_2_3/agc_emis_year/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_aboveground_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/AnnualAbovegroundRemovalFactorCarbon.scala b/src/main/scala/org/globalforestwatch/layers/AnnualAbovegroundRemovalFactorCarbon.scala index 8099ecb3..3b3515e8 100644 --- a/src/main/scala/org/globalforestwatch/layers/AnnualAbovegroundRemovalFactorCarbon.scala +++ b/src/main/scala/org/globalforestwatch/layers/AnnualAbovegroundRemovalFactorCarbon.scala @@ -6,9 +6,15 @@ case class AnnualAbovegroundRemovalFactorCarbon(gridTile: GridTile, model: Strin extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_forest_flux_full_extent_removal_factor_aboveground" - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - s"s3://gfw-files/flux_1_2_3/annual_removal_factor_AGC_all_forest_types/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_full_extent_removal_factor_aboveground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1_yr-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/AnnualBelowgroundRemovalFactorCarbon.scala b/src/main/scala/org/globalforestwatch/layers/AnnualBelowgroundRemovalFactorCarbon.scala index 1bc5842d..5c82fa27 100644 --- a/src/main/scala/org/globalforestwatch/layers/AnnualBelowgroundRemovalFactorCarbon.scala +++ b/src/main/scala/org/globalforestwatch/layers/AnnualBelowgroundRemovalFactorCarbon.scala @@ -6,9 +6,14 @@ case class AnnualBelowgroundRemovalFactorCarbon(gridTile: GridTile, model: Strin extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_forest_flux_full_extent_removal_factor_belowground" - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - s"s3://gfw-files/flux_1_2_3/annual_removal_factor_BGC_all_forest_types/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_full_extent_removal_factor_belowground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1_yr-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/BelowgroundCarbon2000.scala b/src/main/scala/org/globalforestwatch/layers/BelowgroundCarbon2000.scala index f6b8f63a..5f93f6d5 100644 --- a/src/main/scala/org/globalforestwatch/layers/BelowgroundCarbon2000.scala +++ b/src/main/scala/org/globalforestwatch/layers/BelowgroundCarbon2000.scala @@ -6,16 +6,16 @@ case class BelowgroundCarbon2000(gridTile: GridTile, model: String = "standard", extends FloatLayer with OptionalFLayer { - val datasetName = "gfw_belowground_carbon" + val datasetName = "gfw_belowground_carbon_stock_2000" val uri: String = uriForGrid(gridTile, kwargs) -// // For carbonflux package run only +// // For carbon_sensitivity run only (but not currently functional) // val datasetName = "Na" // // val model_suffix: String = if (model == "standard") "standard" else s"$model" // val uri: String = -// s"s3://gfw-files/flux_1_2_3/bgc_2000/$model_suffix/${gridTile.tileId}.tif" +// s"s3://gfw-data-lake/gfw_belowground_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" } \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/layers/BelowgroundCarbonEmisYear.scala b/src/main/scala/org/globalforestwatch/layers/BelowgroundCarbonEmisYear.scala index cc83d9c6..c5477970 100644 --- a/src/main/scala/org/globalforestwatch/layers/BelowgroundCarbonEmisYear.scala +++ b/src/main/scala/org/globalforestwatch/layers/BelowgroundCarbonEmisYear.scala @@ -6,9 +6,14 @@ case class BelowgroundCarbonEmisYear(gridTile: GridTile, model: String = "standa extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_forest_flux_belowground_carbon_stock_in_emissions_year" - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - s"s3://gfw-files/flux_1_2_3/bgc_emis_year/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_belowground_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/DeadwoodCarbon2000.scala b/src/main/scala/org/globalforestwatch/layers/DeadwoodCarbon2000.scala index 21cb3479..6df1f20a 100644 --- a/src/main/scala/org/globalforestwatch/layers/DeadwoodCarbon2000.scala +++ b/src/main/scala/org/globalforestwatch/layers/DeadwoodCarbon2000.scala @@ -6,16 +6,16 @@ case class DeadwoodCarbon2000(gridTile: GridTile, model: String = "standard", kw extends FloatLayer with OptionalFLayer { -// val datasetName = "gfw_deadwood_carbon" -// -// val uri: String = -// uriForGrid(gridTile, kwargs) + val datasetName = "gfw_deadwood_carbon_stock_2000" + val uri: String = + uriForGrid(gridTile, kwargs) - // For carbonflux package run only - val datasetName = "Na" - val model_suffix: String = if (model == "standard") "standard" else s"$model" - val uri: String = - s"s3://gfw-files/flux_1_2_3/deadwood_carbon_2000/$model_suffix/${gridTile.tileId}.tif" +// // For carbon_sensitivity run only (but not currently functional) +// val datasetName = "Na" +// +// val model_suffix: String = if (model == "standard") "standard" else s"$model" +// val uri: String = +// s"s3://gfw-data-lake/gfw_deadwood_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/DeadwoodCarbonEmisYear.scala b/src/main/scala/org/globalforestwatch/layers/DeadwoodCarbonEmisYear.scala index c5f56432..de177870 100644 --- a/src/main/scala/org/globalforestwatch/layers/DeadwoodCarbonEmisYear.scala +++ b/src/main/scala/org/globalforestwatch/layers/DeadwoodCarbonEmisYear.scala @@ -6,9 +6,14 @@ case class DeadwoodCarbonEmisYear(gridTile: GridTile, model: String = "standard" extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_forest_flux_deadwood_carbon_stock_in_emissions_year" - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - s"s3://gfw-files/flux_1_2_3/deadwood_emis_year/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_deadwood_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/FluxModelExtent.scala b/src/main/scala/org/globalforestwatch/layers/FluxModelExtent.scala deleted file mode 100644 index da8f5ea6..00000000 --- a/src/main/scala/org/globalforestwatch/layers/FluxModelExtent.scala +++ /dev/null @@ -1,14 +0,0 @@ -package org.globalforestwatch.layers - -import org.globalforestwatch.grids.GridTile - -case class FluxModelExtent(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) - extends BooleanLayer - with OptionalILayer { - - val datasetName = "Na" - - val model_suffix: String = if (model == "standard") "standard" else s"$model" - val uri: String = - s"s3://gfw-files/flux_1_2_3/model_extent/$model_suffix/${gridTile.tileId}.tif" -} diff --git a/src/main/scala/org/globalforestwatch/layers/ForestAgeCategory.scala b/src/main/scala/org/globalforestwatch/layers/ForestAgeCategory.scala deleted file mode 100644 index b2113a1f..00000000 --- a/src/main/scala/org/globalforestwatch/layers/ForestAgeCategory.scala +++ /dev/null @@ -1,23 +0,0 @@ -package org.globalforestwatch.layers - -import org.globalforestwatch.grids.GridTile - -case class ForestAgeCategory(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) - extends StringLayer - with OptionalILayer { - - val datasetName = "Na" - - val model_suffix: String = if (model == "standard") "standard" else s"$model" - val uri: String = - s"s3://gfw-files/flux_1_2_3/forest_age_category/$model_suffix/${gridTile.tileId}.tif" - - override val externalNoDataValue = "Not applicable" - - def lookup(value: Int): String = value match { - case 1 => "Secondary forest <=20 years" - case 2 => "Secondary forest >20 years" - case 3 => "Primary forest" - case _ => "Unknown" - } -} diff --git a/src/main/scala/org/globalforestwatch/layers/ForestFluxModelAgeCategory.scala b/src/main/scala/org/globalforestwatch/layers/ForestFluxModelAgeCategory.scala new file mode 100644 index 00000000..3e91cbca --- /dev/null +++ b/src/main/scala/org/globalforestwatch/layers/ForestFluxModelAgeCategory.scala @@ -0,0 +1,30 @@ +package org.globalforestwatch.layers + +import org.globalforestwatch.grids.GridTile + +case class ForestFluxModelAgeCategory(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) + extends StringLayer + with OptionalILayer { + + val datasetName = "gfw_forest_flux_forest_age_category" + + val uri: String = + uriForGrid(gridTile, kwargs) + + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_forest_age_category/v20231114/raster/epsg-4326/{grid_size}/{row_count}/ageCategory/geotiff/{tile_id}.tif" + + override val externalNoDataValue = "Not applicable" + + def lookup(value: Int): String = value match { + case 1 => "Secondary forest <=20 years" + case 2 => "Secondary forest >20 years" + case 3 => "Primary forest" + case _ => "Unknown" + } +} diff --git a/src/main/scala/org/globalforestwatch/layers/ForestFluxModelExtent.scala b/src/main/scala/org/globalforestwatch/layers/ForestFluxModelExtent.scala new file mode 100644 index 00000000..79af57cf --- /dev/null +++ b/src/main/scala/org/globalforestwatch/layers/ForestFluxModelExtent.scala @@ -0,0 +1,21 @@ +package org.globalforestwatch.layers + +import org.globalforestwatch.grids.GridTile + +case class ForestFluxModelExtent(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) + extends BooleanLayer + with OptionalILayer { + + val datasetName = "gfw_forest_flux_model_extent" + + val uri: String = + uriForGrid(gridTile, kwargs) + + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_model_extent/v20231114/raster/epsg-4326/{grid_size}/{row_count}/ha/geotiff/{tile_id}.tif" +} diff --git a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNodeCodes.scala b/src/main/scala/org/globalforestwatch/layers/ForestFluxModelGrossEmissionsNodeCodes.scala similarity index 88% rename from src/main/scala/org/globalforestwatch/layers/GrossEmissionsNodeCodes.scala rename to src/main/scala/org/globalforestwatch/layers/ForestFluxModelGrossEmissionsNodeCodes.scala index db0caef0..a5153fda 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNodeCodes.scala +++ b/src/main/scala/org/globalforestwatch/layers/ForestFluxModelGrossEmissionsNodeCodes.scala @@ -1,15 +1,22 @@ package org.globalforestwatch.layers + import org.globalforestwatch.grids.GridTile -case class GrossEmissionsNodeCodes(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) +case class ForestFluxModelGrossEmissionsNodeCodes(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) extends StringLayer with OptionalILayer { - val datasetName = "Na" + val datasetName = "gfw_forest_flux_gross_emissions_node_codes" - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - s"s3://gfw-files/flux_1_2_3/gross_emissions_node_codes/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_gross_emissions_node_codes/v20231114/raster/epsg-4326/{grid_size}/{row_count}/category/geotiff/{tile_id}.tif" override val externalNoDataValue = "Not applicable" diff --git a/src/main/scala/org/globalforestwatch/layers/ForestFluxModelPlantedForestType.scala b/src/main/scala/org/globalforestwatch/layers/ForestFluxModelPlantedForestType.scala new file mode 100644 index 00000000..ec157b93 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/layers/ForestFluxModelPlantedForestType.scala @@ -0,0 +1,30 @@ +package org.globalforestwatch.layers + +import org.globalforestwatch.grids.GridTile + +case class ForestFluxModelPlantedForestType(gridTile: GridTile, kwargs: Map[String, Any]) + extends StringLayer + with OptionalILayer { + + val datasetName = "gfw_forest_flux_planted_forest_type" + + val uri: String = + uriForGrid(gridTile, kwargs) + + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + //// val uri: String = + //// s"s3://gfw-data-lake/gfw_planted_forests/v20230911/raster/epsg-4326/{grid_size}/{row_count}/simpleName/geotiff/{tile_id}.tif" + + override val externalNoDataValue = "Not applicable" + + def lookup(value: Int): String = value match { + case 1 => "Oil palm" + case 2 => "Wood fiber" + case 3 => "Other" + case _ => "Unknown" + } +} diff --git a/src/main/scala/org/globalforestwatch/layers/RemovalForestType.scala b/src/main/scala/org/globalforestwatch/layers/ForestFluxModelRemovalForestType.scala similarity index 51% rename from src/main/scala/org/globalforestwatch/layers/RemovalForestType.scala rename to src/main/scala/org/globalforestwatch/layers/ForestFluxModelRemovalForestType.scala index 26319e2d..8575d8e9 100644 --- a/src/main/scala/org/globalforestwatch/layers/RemovalForestType.scala +++ b/src/main/scala/org/globalforestwatch/layers/ForestFluxModelRemovalForestType.scala @@ -1,16 +1,23 @@ package org.globalforestwatch.layers + import org.globalforestwatch.grids.GridTile -case class RemovalForestType(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) +case class ForestFluxModelRemovalForestType(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) extends StringLayer with OptionalILayer { - val datasetName = "Na" - - val model_suffix: String = if (model == "standard") "standard" else s"$model" + val datasetName = "gfw_forest_flux_removal_forest_type" val uri: String = - s"s3://gfw-files/flux_1_2_3/removal_forest_type/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + //// val uri: String = + //// s"s3://gfw-data-lake/gfw_forest_flux_removal_forest_type/v20231114/v20231114/raster/epsg-4326/{grid_size}/{row_count}/source/geotiff/{tile_id}.tif" override val externalNoDataValue = "Not applicable" diff --git a/src/main/scala/org/globalforestwatch/layers/GrossCumulAbovegroundRemovalsCo2.scala b/src/main/scala/org/globalforestwatch/layers/GrossCumulAbovegroundRemovalsCo2.scala index f01af529..7825a816 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossCumulAbovegroundRemovalsCo2.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossCumulAbovegroundRemovalsCo2.scala @@ -6,15 +6,16 @@ case class GrossCumulAbovegroundRemovalsCo2(gridTile: GridTile, model: String = extends FloatLayer with OptionalFLayer { - val datasetName = "gfw_full_extent_aboveground_gross_removals" + val datasetName = "gfw_forest_flux_full_extent_gross_removals_aboveground" val uri: String = uriForGrid(gridTile, kwargs) -// // For carbonflux package run only + +// // For carbon_sensitivity run only (but not currently functional) // val datasetName = "Na" // // val model_suffix: String = if (model == "standard") "standard" else s"$model" // val uri: String = -// s"s3://gfw-files/flux_1_2_3/gross_removals_AGCO2_all_forest_types/$model_suffix/${gridTile.tileId}.tif" +// s"s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_removals_aboveground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/GrossCumulBelowgroundRemovalsCo2.scala b/src/main/scala/org/globalforestwatch/layers/GrossCumulBelowgroundRemovalsCo2.scala index f7e48f62..cc1d3756 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossCumulBelowgroundRemovalsCo2.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossCumulBelowgroundRemovalsCo2.scala @@ -6,16 +6,16 @@ case class GrossCumulBelowgroundRemovalsCo2(gridTile: GridTile, model: String = extends FloatLayer with OptionalFLayer { - val datasetName = "gfw_full_extent_belowground_gross_removals" + val datasetName = "gfw_forest_flux_full_extent_gross_removals_belowground" val uri: String = uriForGrid(gridTile, kwargs) -// // For carbonflux package run only +// // For carbon_sensitivity run only (but not currently functional) // val datasetName = "Na" // // val model_suffix: String = if (model == "standard") "standard" else s"$model" // val uri: String = -// s"s3://gfw-files/flux_1_2_3/gross_removals_BGCO2_all_forest_types/$model_suffix/${gridTile.tileId}.tif" +// s"s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_removals_belowground/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2BiomassSoil.scala b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2BiomassSoil.scala index decf366b..43274ab2 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2BiomassSoil.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2BiomassSoil.scala @@ -6,14 +6,15 @@ case class GrossEmissionsCo2OnlyCo2BiomassSoil(gridTile: GridTile, model: String extends FloatLayer with OptionalFLayer { - val datasetName = "gfw_full_extent_co2_gross_emissions_biomass_soil" + val datasetName = "gfw_forest_flux_full_extent_gross_emissions_co2_only_biomass_soil" val uri: String = uriForGrid(gridTile, kwargs) + // // For carbon_sensitivity run only (but not currently functional) // val datasetName = "Na" // // val model_suffix: String = if (model == "standard") "standard" else s"$model" // val uri: String = - // s"s3://gfw-files/flux_1_2_3/gross_emissions_co2_only_co2e/$model_suffix/${gridTile.tileId}.tif" + // s"s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_co2_only_biomass_soil/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2SoilOnly.scala b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2SoilOnly.scala index 555ea764..70d28e13 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2SoilOnly.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2SoilOnly.scala @@ -6,9 +6,16 @@ case class GrossEmissionsCo2OnlyCo2SoilOnly(gridTile: GridTile, kwargs: Map[Stri extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_forest_flux_full_extent_gross_emissions_co2_only_soil_only" val uri: String = - s"s3://gfw-files/flux_1_2_3/gross_emissions_co2_only_co2e/soil_only/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_co2_only_soil_only/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2eBiomassSoil.scala b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2eBiomassSoil.scala index 3db07051..1cc28f54 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2eBiomassSoil.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2eBiomassSoil.scala @@ -6,16 +6,15 @@ case class GrossEmissionsNonCo2Co2eBiomassSoil(gridTile: GridTile, model: String extends FloatLayer with OptionalFLayer { - val datasetName = "gfw_full_extent_non_co2_gross_emissions_biomass_soil" + val datasetName = "gfw_forest_flux_full_extent_gross_emissions_non_co2_biomass_soil" val uri: String = uriForGrid(gridTile, kwargs) - - // // For carbonflux package run only + // // For carbon_sensitivity run only (but not currently functional) // val datasetName = "Na" // // val model_suffix: String = if (model == "standard") "standard" else s"$model" // val uri: String = - // s"s3://gfw-files/flux_1_2_3/gross_emissions_non_co2_co2e/$model_suffix/${gridTile.tileId}.tif" + // s"s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_non_co2_biomass_soil/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2eSoilOnly.scala b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2eSoilOnly.scala index aecef05e..a5d6cdb7 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2eSoilOnly.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2eSoilOnly.scala @@ -6,8 +6,15 @@ case class GrossEmissionsNonCo2Co2eSoilOnly(gridTile: GridTile, kwargs: Map[Stri extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_forest_flux_full_extent_gross_emissions_non_co2_soil_only" val uri: String = - s"s3://gfw-files/flux_1_2_3/gross_emissions_non_co2_co2e/soil_only/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_full_extent_gross_emissions_non_co2_soil_only/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" } \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/layers/LitterCarbon2000.scala b/src/main/scala/org/globalforestwatch/layers/LitterCarbon2000.scala index c832b7ce..21b6c66e 100644 --- a/src/main/scala/org/globalforestwatch/layers/LitterCarbon2000.scala +++ b/src/main/scala/org/globalforestwatch/layers/LitterCarbon2000.scala @@ -6,16 +6,16 @@ case class LitterCarbon2000(gridTile: GridTile, model: String = "standard", kwar extends FloatLayer with OptionalFLayer { -// val datasetName = "gfw_litter_carbon" -// -// val uri: String = -// uriForGrid(gridTile, kwargs) + val datasetName = "gfw_litter_carbon_stock_2000" + val uri: String = + uriForGrid(gridTile, kwargs) - // For carbonflux package run only - val datasetName = "Na" - val model_suffix: String = if (model == "standard") "standard" else s"$model" - val uri: String = - s"s3://gfw-files/flux_1_2_3/litter_carbon_2000/$model_suffix/${gridTile.tileId}.tif" +// // For carbon_sensitivity run only (but not currently functional) +// val datasetName = "Na" +// +// val model_suffix: String = if (model == "standard") "standard" else s"$model" +// val uri: String = +// s"s3://gfw-data-lake/gfw_litter_carbon_stock_2000/v20230222/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/LitterCarbonEmisYear.scala b/src/main/scala/org/globalforestwatch/layers/LitterCarbonEmisYear.scala index 35942df5..7730e304 100644 --- a/src/main/scala/org/globalforestwatch/layers/LitterCarbonEmisYear.scala +++ b/src/main/scala/org/globalforestwatch/layers/LitterCarbonEmisYear.scala @@ -6,9 +6,12 @@ case class LitterCarbonEmisYear(gridTile: GridTile, model: String = "standard", extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_forest_flux_litter_carbon_stock_in_emissions_year" - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - s"s3://gfw-files/flux_1_2_3/litter_emis_year/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + +// val model_suffix: String = if (model == "standard") "standard" else s"$model" +// val uri: String = +// s"s3://gfw-data-lake/gfw_forest_flux_litter_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/NetFluxCo2e.scala b/src/main/scala/org/globalforestwatch/layers/NetFluxCo2e.scala index 9e613e5f..9b7f08a9 100644 --- a/src/main/scala/org/globalforestwatch/layers/NetFluxCo2e.scala +++ b/src/main/scala/org/globalforestwatch/layers/NetFluxCo2e.scala @@ -6,16 +6,16 @@ case class NetFluxCo2e(gridTile: GridTile, model: String = "standard", kwargs: M extends FloatLayer with OptionalFLayer { - val datasetName = "gfw_full_extent_net_flux" + val datasetName = "gfw_forest_flux_full_extent_net_flux" val uri: String = uriForGrid(gridTile, kwargs) -// // For carbonflux package run only +// // For carbon_sensitivity run only (but not currently functional) // val datasetName = "Na" // // val model_suffix: String = if (model == "standard") "standard" else s"$model" // val uri: String = -// s"s3://gfw-files/flux_1_2_3/net_flux_all_forest_types_all_drivers/$model_suffix/${gridTile.tileId}.tif" +// s"s3://gfw-data-lake/gfw_forest_flux_full_extent_net_flux/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/PlantationsTypeFluxModel.scala b/src/main/scala/org/globalforestwatch/layers/PlantationsTypeFluxModel.scala deleted file mode 100644 index 8238014f..00000000 --- a/src/main/scala/org/globalforestwatch/layers/PlantationsTypeFluxModel.scala +++ /dev/null @@ -1,21 +0,0 @@ -package org.globalforestwatch.layers -import org.globalforestwatch.grids.GridTile - -case class PlantationsTypeFluxModel(gridTile: GridTile, kwargs: Map[String, Any]) - extends StringLayer - with OptionalILayer { - - val datasetName = "Na" - - val uri: String = - s"s3://gfw-files/flux_1_2_0/plantation_type/standard/${gridTile.tileId}.tif" - - override val externalNoDataValue = "Not applicable" - - def lookup(value: Int): String = value match { - case 1 => "Oil palm" - case 2 => "Wood fiber" - case 3 => "Other" - case _ => "Unknown" - } -} diff --git a/src/main/scala/org/globalforestwatch/layers/SoilCarbon2000.scala b/src/main/scala/org/globalforestwatch/layers/SoilCarbon2000.scala index a5a9b713..35545027 100644 --- a/src/main/scala/org/globalforestwatch/layers/SoilCarbon2000.scala +++ b/src/main/scala/org/globalforestwatch/layers/SoilCarbon2000.scala @@ -6,16 +6,16 @@ case class SoilCarbon2000(gridTile: GridTile, model: String = "standard", kwargs extends FloatLayer with OptionalFLayer { - val datasetName = "gfw_soil_carbon" + val datasetName = "gfw_soil_carbon_stock_2000" val uri: String = uriForGrid(gridTile, kwargs) -// // For carbonflux package run only +// // For carbon_sensitivity run only (but not currently functional) // val datasetName = "Na" // // val model_suffix: String = if (model == "standard") "standard" else s"$model" // val uri: String = -// s"s3://gfw-files/flux_1_2_3/soil_carbon_2000/standard/${gridTile.tileId}.tif" +// s"s3://gfw-data-lake/gfw_soil_carbon_stock_2000/v20231108/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/SoilCarbonEmisYear.scala b/src/main/scala/org/globalforestwatch/layers/SoilCarbonEmisYear.scala index 98e6b590..eef2cae8 100644 --- a/src/main/scala/org/globalforestwatch/layers/SoilCarbonEmisYear.scala +++ b/src/main/scala/org/globalforestwatch/layers/SoilCarbonEmisYear.scala @@ -6,9 +6,15 @@ case class SoilCarbonEmisYear(gridTile: GridTile, model: String = "standard", kw extends FloatLayer with OptionalFLayer { - val datasetName = "Na" - val model_suffix: String = if (model == "standard") "standard" else s"$model" + val datasetName = "gfw_forest_flux_soil_carbon_stock_in_emissions_year" + val uri: String = - s"s3://gfw-files/flux_1_2_3/soil_emis_year/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + +// val datasetName = "Na" +// +// val model_suffix: String = if (model == "standard") "standard" else s"$model" +// val uri: String = +// s"s3://gfw-data-lake/gfw_forest_flux_soil_carbon_stock_in_emissions_year/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/StdevAnnualAbovegroundRemovalsCarbon.scala b/src/main/scala/org/globalforestwatch/layers/StdevAnnualAbovegroundRemovalsCarbon.scala index bb1a675b..d37deb2f 100644 --- a/src/main/scala/org/globalforestwatch/layers/StdevAnnualAbovegroundRemovalsCarbon.scala +++ b/src/main/scala/org/globalforestwatch/layers/StdevAnnualAbovegroundRemovalsCarbon.scala @@ -6,9 +6,17 @@ case class StdevAnnualAbovegroundRemovalsCarbon(gridTile: GridTile, model: Stri extends FloatLayer with OptionalFLayer { - val datasetName = "Na" - val model_suffix: String = if (model == "standard") "standard" else s"$model" + val datasetName = "gfw_forest_flux_removal_factor_aboveground_carbon_stdev" + val uri: String = - s"s3://gfw-files/flux_1_2_3/stdev_annual_removal_factor_AGC_all_forest_types/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_removal_factor_aboveground_carbon_stdev/v20231114/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1_yr-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/StdevSoilCarbon2000.scala b/src/main/scala/org/globalforestwatch/layers/StdevSoilCarbon2000.scala index cecd245c..07a7c515 100644 --- a/src/main/scala/org/globalforestwatch/layers/StdevSoilCarbon2000.scala +++ b/src/main/scala/org/globalforestwatch/layers/StdevSoilCarbon2000.scala @@ -6,10 +6,16 @@ case class StdevSoilCarbon2000(gridTile: GridTile, model: String = "standard", k extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_forest_flux_soil_carbon_stock_2000_stdev" - - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - s"s3://gfw-files/flux_1_2_0/stdev_soil_carbon_full_extent/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile, kwargs) + + + // // For carbon_sensitivity run only (but not currently functional) + // val datasetName = "Na" + // + // val model_suffix: String = if (model == "standard") "standard" else s"$model" + // val uri: String = + // s"s3://gfw-data-lake/gfw_forest_flux_soil_carbon_stock_2000_stdev/v20231108/raster/epsg-4326/{grid_size}/{row_count}/Mg_C_ha-1/geotiff/{tile_id}.tif" } diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala index f28a71ac..3b000114 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala @@ -29,8 +29,8 @@ case class CarbonSensitivityGridSources(gridTile: GridTile, val grossEmissionsCo2eNonCo2: GrossEmissionsNonCo2Co2eBiomassSoil = GrossEmissionsNonCo2Co2eBiomassSoil(gridTile, model, kwargs) val grossEmissionsCo2eCo2Only: GrossEmissionsCo2OnlyCo2BiomassSoil = GrossEmissionsCo2OnlyCo2BiomassSoil(gridTile, model, kwargs) val jplTropicsAbovegroundBiomassDensity2000: JplTropicsAbovegroundBiomassDensity2000 = JplTropicsAbovegroundBiomassDensity2000(gridTile, kwargs) - val fluxModelExtent: FluxModelExtent = FluxModelExtent(gridTile, model, kwargs) - val removalForestType: RemovalForestType = RemovalForestType(gridTile, model, kwargs) + val fluxModelExtent: ForestFluxModelExtent = ForestFluxModelExtent(gridTile, model, kwargs) + val removalForestType: ForestFluxModelRemovalForestType = ForestFluxModelRemovalForestType(gridTile, model, kwargs) val treeCoverGain: TreeCoverGain = TreeCoverGain(gridTile, kwargs) val mangroveBiomassExtent: MangroveBiomassExtent = MangroveBiomassExtent(gridTile, kwargs) val treeCoverLossDrivers: TreeCoverLossDrivers = TreeCoverLossDrivers(gridTile, kwargs) @@ -38,10 +38,10 @@ case class CarbonSensitivityGridSources(gridTile: GridTile, val protectedAreas: ProtectedAreas = ProtectedAreas(gridTile, kwargs) val landmark: Landmark = Landmark(gridTile, kwargs) val intactForestLandscapes: IntactForestLandscapes = IntactForestLandscapes(gridTile, kwargs) - val plantationsTypeFluxModel: PlantationsTypeFluxModel = PlantationsTypeFluxModel(gridTile, kwargs) + val plantationsTypeFluxModel: ForestFluxModelPlantedForestType = ForestFluxModelPlantedForestType(gridTile, kwargs) val intactPrimaryForest: IntactPrimaryForest = IntactPrimaryForest(gridTile, kwargs) val peatlandsExtentFluxModel: Peatlands = Peatlands(gridTile, kwargs) - val forestAgeCategory: ForestAgeCategory = ForestAgeCategory(gridTile, model, kwargs) + val forestAgeCategory: ForestFluxModelAgeCategory = ForestFluxModelAgeCategory(gridTile, model, kwargs) val jplTropicsAbovegroundBiomassExtent2000: JplTropicsAbovegroundBiomassExtent2000 = JplTropicsAbovegroundBiomassExtent2000(gridTile, kwargs) val fiaRegionsUsExtent: FiaRegionsUsExtent = FiaRegionsUsExtent(gridTile, kwargs) @@ -53,7 +53,7 @@ case class CarbonSensitivityGridSources(gridTile: GridTile, ProdesLegalAmazonExtent2000(gridTile, kwargs) val tropicLatitudeExtent: TropicLatitudeExtent = TropicLatitudeExtent(gridTile, kwargs) val treeCoverLossFromFires: TreeCoverLossFromFires = TreeCoverLossFromFires(gridTile, kwargs) - val grossEmissionsNodeCodes: GrossEmissionsNodeCodes = GrossEmissionsNodeCodes(gridTile, model, kwargs) + val grossEmissionsNodeCodes: ForestFluxModelGrossEmissionsNodeCodes = ForestFluxModelGrossEmissionsNodeCodes(gridTile, model, kwargs) def readWindow( windowKey: SpatialKey, diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityTile.scala b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityTile.scala index a75ce97b..d4d40861 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityTile.scala @@ -22,18 +22,18 @@ case class CarbonSensitivityTile( grossEmissionsCo2eCo2Only: GrossEmissionsCo2OnlyCo2BiomassSoil#OptionalFTile, jplTropicsAbovegroundBiomassDensity2000: JplTropicsAbovegroundBiomassDensity2000#OptionalFTile, - fluxModelExtent: FluxModelExtent#OptionalITile, - removalForestType: RemovalForestType#OptionalITile, + fluxModelExtent: ForestFluxModelExtent#OptionalITile, + removalForestType: ForestFluxModelRemovalForestType#OptionalITile, mangroveBiomassExtent: MangroveBiomassExtent#OptionalITile, drivers: TreeCoverLossDrivers#OptionalITile, ecozones: FaoEcozones2000#OptionalITile, landmark: Landmark#OptionalITile, wdpa: ProtectedAreas#OptionalITile, intactForestLandscapes: IntactForestLandscapes#OptionalITile, - plantationsTypeFluxModel: PlantationsTypeFluxModel#OptionalITile, + plantationsTypeFluxModel: ForestFluxModelPlantedForestType#OptionalITile, intactPrimaryForest: IntactPrimaryForest#OptionalITile, peatlandsExtentFluxModel: Peatlands#OptionalITile, - forestAgeCategory: ForestAgeCategory#OptionalITile, + forestAgeCategory: ForestFluxModelAgeCategory#OptionalITile, jplTropicsAbovegroundBiomassExtent2000: JplTropicsAbovegroundBiomassExtent2000#OptionalITile, fiaRegionsUsExtent: FiaRegionsUsExtent#OptionalITile, braBiomes: BrazilBiomes#OptionalITile, @@ -43,7 +43,7 @@ case class CarbonSensitivityTile( prodesLegalAmazonExtent2000: ProdesLegalAmazonExtent2000#OptionalITile, tropicLatitudeExtent: TropicLatitudeExtent#OptionalITile, treeCoverLossFromFires: TreeCoverLossFromFires#OptionalITile, - grossEmissionsNodeCodes: GrossEmissionsNodeCodes#OptionalITile + grossEmissionsNodeCodes: ForestFluxModelGrossEmissionsNodeCodes#OptionalITile ) extends CellGrid[Int] { def cellType: CellType = loss.cellType diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxExport.scala b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxExport.scala index 7dce31f0..b958690a 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxExport.scala @@ -101,7 +101,7 @@ object CarbonFluxExport extends SummaryExport { .filter($"umd_tree_cover_loss__year".isNotNull && ($"umd_tree_cover_loss__ha" > 0 || $"gfw_full_extent_gross_emissions_biomass_soil__Mg_CO2e" > 0)) .transform(CarbonFluxDF.aggChange(List("iso", "adm1", "adm2"))) - .coalesce(200) // this should result in an avg file size of 50MB. We try to keep filesize small due to memory issues + .coalesce(300) // this should result in an avg file size of 50MB. We try to keep filesize small due to memory issues // .coalesce(1) // for local testing: produces one output csv @@ -111,7 +111,7 @@ object CarbonFluxExport extends SummaryExport { val adm1ApiDF = adm2ApiDF .transform(CarbonFluxDF.aggChange(List("iso", "adm1"))) - .coalesce(120) // this should result in an avg file size of 50MB. We try to keep filesize small due to memory issues + .coalesce(180) // this should result in an avg file size of 50MB. We try to keep filesize small due to memory issues // .coalesce(1) // for local testing: produces one output csv @@ -121,7 +121,7 @@ object CarbonFluxExport extends SummaryExport { val isoApiDF = adm1ApiDF .transform(CarbonFluxDF.aggChange(List("iso"))) - .coalesce(40) // this should result in an avg file size of 50MB. We try to keep filesize small due to memory issues + .coalesce(60) // this should result in an avg file size of 50MB. We try to keep filesize small due to memory issues // .coalesce(1) // for local testing: produces one output csv diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala index 1d981b44..37a8a10e 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala @@ -39,18 +39,18 @@ case class CarbonFluxGridSources(gridTile: GridTile, kwargs: Map[String, Any]) val stdevAnnualAbovegroundRemovalsCarbon: StdevAnnualAbovegroundRemovalsCarbon = StdevAnnualAbovegroundRemovalsCarbon(gridTile, kwargs = kwargs) val stdevSoilCarbon2000: StdevSoilCarbon2000 = StdevSoilCarbon2000(gridTile, kwargs = kwargs) - val fluxModelExtent: FluxModelExtent = FluxModelExtent(gridTile, kwargs = kwargs) + val fluxModelExtent: ForestFluxModelExtent = ForestFluxModelExtent(gridTile, kwargs = kwargs) val mangroveBiomassExtent: MangroveBiomassExtent = MangroveBiomassExtent(gridTile, kwargs) - val removalForestType: RemovalForestType = RemovalForestType(gridTile, kwargs = kwargs) + val removalForestType: ForestFluxModelRemovalForestType = ForestFluxModelRemovalForestType(gridTile, kwargs = kwargs) val treeCoverLossDrivers: TreeCoverLossDrivers = TreeCoverLossDrivers(gridTile, kwargs) val faoEcozones2000: FaoEcozones2000 = FaoEcozones2000(gridTile, kwargs = kwargs) val protectedAreas: ProtectedAreas = ProtectedAreas(gridTile, kwargs) val landmark: Landmark = Landmark(gridTile, kwargs) val intactForestLandscapes2000: IntactForestLandscapes2000 = IntactForestLandscapes2000(gridTile, kwargs) - val plantationsTypeFluxModel: PlantationsTypeFluxModel = PlantationsTypeFluxModel(gridTile, kwargs) + val plantationsTypeFluxModel: ForestFluxModelPlantedForestType = ForestFluxModelPlantedForestType(gridTile, kwargs) val intactPrimaryForest: IntactPrimaryForest = IntactPrimaryForest(gridTile, kwargs) val peatlands: Peatlands = Peatlands(gridTile, kwargs) - val forestAgeCategory: ForestAgeCategory = ForestAgeCategory(gridTile, kwargs = kwargs) + val forestAgeCategory: ForestFluxModelAgeCategory = ForestFluxModelAgeCategory(gridTile, kwargs = kwargs) val jplTropicsAbovegroundBiomassExtent2000: JplTropicsAbovegroundBiomassExtent2000 = JplTropicsAbovegroundBiomassExtent2000(gridTile, kwargs) val fiaRegionsUsExtent: FiaRegionsUsExtent = FiaRegionsUsExtent(gridTile, kwargs) val brazilBiomes: BrazilBiomes = BrazilBiomes(gridTile, kwargs) @@ -60,7 +60,7 @@ case class CarbonFluxGridSources(gridTile: GridTile, kwargs: Map[String, Any]) val prodesLegalAmazonExtent2000: ProdesLegalAmazonExtent2000 = ProdesLegalAmazonExtent2000(gridTile, kwargs) val tropicLatitudeExtent: TropicLatitudeExtent = TropicLatitudeExtent(gridTile, kwargs) val treeCoverLossFromFires: TreeCoverLossFromFires = TreeCoverLossFromFires(gridTile, kwargs) - val grossEmissionsNodeCodes: GrossEmissionsNodeCodes = GrossEmissionsNodeCodes(gridTile, kwargs = kwargs) + val grossEmissionsNodeCodes: ForestFluxModelGrossEmissionsNodeCodes = ForestFluxModelGrossEmissionsNodeCodes(gridTile, kwargs = kwargs) val plantationsPre2000: PlantationsPre2000 = PlantationsPre2000(gridTile, kwargs) def readWindow( diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxTile.scala b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxTile.scala index 85858a16..325e5a2a 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxTile.scala @@ -36,18 +36,18 @@ case class CarbonFluxTile( stdevAnnualAbovegroundRemovalsCarbon: StdevAnnualAbovegroundRemovalsCarbon#OptionalFTile, stdevSoilCarbon2000: StdevSoilCarbon2000#OptionalFTile, - fluxModelExtent: FluxModelExtent#OptionalITile, + fluxModelExtent: ForestFluxModelExtent#OptionalITile, mangroveBiomassExtent: MangroveBiomassExtent#OptionalITile, - removalForestType: RemovalForestType#OptionalITile, + removalForestType: ForestFluxModelRemovalForestType#OptionalITile, drivers: TreeCoverLossDrivers#OptionalITile, faoEcozones2000: FaoEcozones2000#OptionalITile, landmark: Landmark#OptionalITile, wdpa: ProtectedAreas#OptionalITile, intactForestLandscapes2000: IntactForestLandscapes2000#OptionalITile, - plantationsTypeFluxModel: PlantationsTypeFluxModel#OptionalITile, + plantationsTypeFluxModel: ForestFluxModelPlantedForestType#OptionalITile, intactPrimaryForest: IntactPrimaryForest#OptionalITile, peatlands: Peatlands#OptionalITile, - forestAgeCategory: ForestAgeCategory#OptionalITile, + forestAgeCategory: ForestFluxModelAgeCategory#OptionalITile, jplTropicsAbovegroundBiomassExtent2000: JplTropicsAbovegroundBiomassExtent2000#OptionalITile, fiaRegionsUsExtent: FiaRegionsUsExtent#OptionalITile, brazilBiomes: BrazilBiomes#OptionalITile, @@ -57,7 +57,7 @@ case class CarbonFluxTile( prodesLegalAmazonExtent2000: ProdesLegalAmazonExtent2000#OptionalITile, tropicLatitudeExtent: TropicLatitudeExtent#OptionalITile, treeCoverLossFromFires: TreeCoverLossFromFires#OptionalITile, - grossEmissionsNodeCodes: GrossEmissionsNodeCodes#OptionalITile, + grossEmissionsNodeCodes: ForestFluxModelGrossEmissionsNodeCodes#OptionalITile, plantationsPre2000: PlantationsPre2000#OptionalITile ) extends CellGrid[Int] { def cellType: CellType = loss.cellType diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala index 8713a4b7..4399ad4f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala @@ -32,7 +32,7 @@ case class TreeLossGridSources(gridTile: GridTile, kwargs: Map[String, Any]) ext val grossEmissionsCo2eNonCo2: GrossEmissionsNonCo2Co2eBiomassSoil = GrossEmissionsNonCo2Co2eBiomassSoil(gridTile, kwargs = kwargs) val grossEmissionsCo2eCo2Only: GrossEmissionsCo2OnlyCo2BiomassSoil = GrossEmissionsCo2OnlyCo2BiomassSoil(gridTile, kwargs = kwargs) val netFluxCo2e: NetFluxCo2e = NetFluxCo2e(gridTile, kwargs = kwargs) - val fluxModelExtent: FluxModelExtent = FluxModelExtent(gridTile, kwargs = kwargs) + val fluxModelExtent: ForestFluxModelExtent = ForestFluxModelExtent(gridTile, kwargs = kwargs) def readWindow(windowKey: SpatialKey, windowLayout: LayoutDefinition):