From 3713c9924bd2b1b3d56b2cb956609cdb062dc047 Mon Sep 17 00:00:00 2001 From: Dan Scales Date: Wed, 12 Jun 2024 10:33:34 -0700 Subject: [PATCH] GTC-2826 Colombia-specific analysis in forest_change_diagnostic We now do country-specific analyses for Brazil (BRA), Argentina (ARG), and Colombia (COL). Updated the gfwpro_forest_change_regions dataset to new v20240529 version that includes a bit indicating a pixel location is in COL, so we can do the necessary Colombia-specific analysis. Added a Colombia country-specific classification from the new Colombia col_frontera_agricola/v2024 dataset that I added. For Argentina, the analogous country-specific classification is OTBN. For the GFWProCoverage layer, I changed the code to just return the actual sets of bits, rather than a map corresponding to all the 1 bits. It seems very inefficient to create a new Map object for every single pixel being scanned, whereas a bitmap with access functions is very efficient. New columns: - country_area - amount of area of location in each of BRA, ARG, and COL. Tells the front-end whether a country-specific tab needs to be shown for this location/list. - classified_region_area - for COL/ARG only - area breakdown by COL (Frontera agricola) or ARG (OBTN) classifications. Eventually arg_obtn_area will be removed, since replaced by this more general column. - tree_cover_loss_by_country_classified_region_yearly - for COL only, TCL by Frontier Agricola categories and by year Existing columns that are extended to COL - protected_areas_by_category_area - for ARG already and now COL - landmark_by_category_area - for ARG already and now COL - tree_cover_loss_by_country_yearly - for ARG and BRA already and now COL - tree_cover_loss_by_country_wdpa_yearly - for ARG and BRA already and now COL - tree_cover_loss_by_country_landmark_yearly - for ARG and BRA already and now COL To support classified_region_area, I had to add a new two-level categorization type, ForestChangeDiagnosticDataDoubleTwoCategory (i.e. there are two levels of categorization by strings, and the final value is a Double). Added extra include checks in the fill operations of several of these data types, so we don't have an empty country entry when there is no forest loss, protected area, landmark, etc for a particular country. Added a new Colombia-specific test. (Removed "*.tsv" from .gitignore - makes it confusing when you want to add new tsv input files.) --- .gitignore | 1 - src/main/resources/raster-catalog-pro.json | 6 +- .../layers/ColFronteraAgricola.scala | 27 ++++++++ .../layers/GFWProCoverage.scala | 27 ++++---- .../ForestChangeDiagnosticDF.scala | 3 + .../ForestChangeDiagnosticData.scala | 25 ++++++-- ...stChangeDiagnosticDataDoubleCategory.scala | 14 ++--- ...hangeDiagnosticDataDoubleTwoCategory.scala | 62 +++++++++++++++++++ ...agnosticDataLossApproxYearlyCategory.scala | 2 +- ...osticDataLossApproxYearlyTwoCategory.scala | 2 +- ...angeDiagnosticDataLossYearlyCategory.scala | 2 +- ...eDiagnosticDataLossYearlyTwoCategory.scala | 2 +- .../ForestChangeDiagnosticGridSources.scala | 1 + .../ForestChangeDiagnosticRawDataGroup.scala | 24 ++++++- .../ForestChangeDiagnosticSummary.scala | 43 +++++++------ .../ForestChangeDiagnosticTile.scala | 1 + .../forest_change_diagnostic/package.scala | 3 + ...39776-5c3b-4db8-8cac-3d284de2cca7-c000.csv | 2 + ...45206-7d33-484d-bead-858f0b3ea115-c000.csv | 2 - ...90df8-9a99-416f-a597-d2d82cef5ee8-c000.csv | 3 + src/test/resources/colombia.tsv | 3 + ...b10f2-c761-4468-b2f2-ac1ce38059ae-c000.csv | 2 - ...4ff7e-a350-48c1-8647-6ab1b9c184da-c000.csv | 2 + .../ForestChangeDiagnosticAnalysisSpec.scala | 18 ++++++ 24 files changed, 222 insertions(+), 55 deletions(-) create mode 100644 src/main/scala/org/globalforestwatch/layers/ColFronteraAgricola.scala create mode 100644 src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleTwoCategory.scala create mode 100644 src/test/resources/argbra-fcd-output/part-00000-3d339776-5c3b-4db8-8cac-3d284de2cca7-c000.csv delete mode 100644 src/test/resources/argbra-fcd-output/part-00000-95345206-7d33-484d-bead-858f0b3ea115-c000.csv create mode 100644 src/test/resources/col-fcd-output/part-00000-97890df8-9a99-416f-a597-d2d82cef5ee8-c000.csv create mode 100755 src/test/resources/colombia.tsv delete mode 100644 src/test/resources/palm-32-fcd-output/part-00000-620b10f2-c761-4468-b2f2-ac1ce38059ae-c000.csv create mode 100644 src/test/resources/palm-32-fcd-output/part-00000-f0e4ff7e-a350-48c1-8647-6ab1b9c184da-c000.csv diff --git a/.gitignore b/.gitignore index 8b9a2b58..43dd4b7a 100644 --- a/.gitignore +++ b/.gitignore @@ -54,4 +54,3 @@ nohup.out derby.log metastore_db/ *.log -*.tsv diff --git a/src/main/resources/raster-catalog-pro.json b/src/main/resources/raster-catalog-pro.json index e7728b36..3339600b 100755 --- a/src/main/resources/raster-catalog-pro.json +++ b/src/main/resources/raster-catalog-pro.json @@ -222,7 +222,7 @@ }, { "name":"gfwpro_forest_change_regions", - "source_uri": "s3://gfw-data-lake/gfwpro_forest_change_regions/v20230207/raster/epsg-4326/{grid_size}/{row_count}/bit_encoding/gdal-geotiff/{tile_id}.tif" + "source_uri": "s3://gfw-data-lake/gfwpro_forest_change_regions/v20240529/raster/epsg-4326/{grid_size}/{row_count}/bit_encoding/gdal-geotiff/{tile_id}.tif" }, { "name":"ifl_intact_forest_landscapes_2000", @@ -263,6 +263,10 @@ { "name":"jrc_global_forest_cover", "source_uri":"s3://gfw-data-lake/jrc_global_forest_cover/v2020/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif" + }, + { + "name":"col_frontera_agricola", + "source_uri":"s3://gfw-data-lake/col_frontera_agricola/v2024/raster/epsg-4326/{grid_size}/{row_count}/category/gdal-geotiff/{tile_id}.tif" } ] } diff --git a/src/main/scala/org/globalforestwatch/layers/ColFronteraAgricola.scala b/src/main/scala/org/globalforestwatch/layers/ColFronteraAgricola.scala new file mode 100644 index 00000000..48751cf3 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/layers/ColFronteraAgricola.scala @@ -0,0 +1,27 @@ +package org.globalforestwatch.layers + +import org.globalforestwatch.grids.GridTile + +// Colombia classification of all its lands into areas where agricultural activity is +// allowed, or is conditional, or is fully restricted. The classification strings are +// intended to be parsed and used directly by the front-end (so front end doesn't +// have to have country-specific information in it). +case class ColFronteraAgricola(gridTile: GridTile, kwargs: Map[String, Any]) extends StringLayer with OptionalILayer { + val datasetName = "col_frontera_agricola" + val uri: String = uriForGrid(gridTile, kwargs) + + def lookup(value: Int): String = value match { + case 1 => "No condicionado;Frontera agricola nacional" + case 2 => "Condicionado;Ambiental" + case 3 => "Condicionado;Ambiental/Riesgo de desastres" + case 4 => "Condicionado;Ambiental/Riesgo de desastres/Étnico-Cultural'" + case 5 => "Condicionado;Ambiental/Étnico-Cultural" + case 6 => "Condicionado;Gestión riesgo de desastres" + case 7 => "Condicionado;Riesgo de desastres/Étnico-Cultural" + case 8 => "Condicionado;Étnico-Cultural" + case 9 => "Zona restricción;Restricciones acuerdo cero deforestación" + case 10 => "Zona restricción;Restricciones legales" + case 11 => "Zona restricción;Restricciones técnicas (Áreas no agropecuarias)" + case _ => "" + } +} diff --git a/src/main/scala/org/globalforestwatch/layers/GFWProCoverage.scala b/src/main/scala/org/globalforestwatch/layers/GFWProCoverage.scala index fd8aa891..702db7c5 100644 --- a/src/main/scala/org/globalforestwatch/layers/GFWProCoverage.scala +++ b/src/main/scala/org/globalforestwatch/layers/GFWProCoverage.scala @@ -3,25 +3,24 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile case class GFWProCoverage(gridTile: GridTile, kwargs: Map[String, Any]) - extends MapILayer + extends IntegerLayer with OptionalILayer { val datasetName = "gfwpro_forest_change_regions" val uri: String = uriForGrid(gridTile, kwargs) +} - def lookup(value: Int): Map[String, Boolean] = { - val bits = "000000000" + value.toBinaryString takeRight 9 - Map( - "Argentina" -> (bits(8) == '1'), - "Soy Coverage" -> (bits(7) == '1'), - "South America" -> (bits(6) == '1'), - "Legal Amazon" -> (bits(5) == '1'), - "Brazil Biomes" -> (bits(4) == '1'), - "Cerrado Biomes" -> (bits(3) == '1'), - "South East Asia" -> (bits(2) == '1'), - "Indonesia" -> (bits(1) == '1') - ) - } +object GFWProCoverage { + def isArgentina(v: Integer): Boolean = (v & (1 << 0)) != 0 + def isColombia(v: Integer): Boolean = (v & (1 << 1)) != 0 + def isSouthAmerica(v: Integer): Boolean = (v & (1 << 2)) != 0 + def isLegalAmazonPresence(v: Integer): Boolean = (v & (1 << 3)) != 0 + // Brazil Biomes presence is true for all Brazil, except for a few narrow costal + // areas. + def isBrazilBiomesPresence(v: Integer): Boolean = (v & (1 << 4)) != 0 + def isCerradoBiomesPresence(v: Integer): Boolean = (v & (1 << 5)) != 0 + def isSouthEastAsia(v: Integer): Boolean = (v & (1 << 6)) != 0 + def isIndonesia(v: Integer): Boolean = (v & (1 << 7)) != 0 } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDF.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDF.scala index 5acda4bc..71ba6a66 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDF.scala @@ -98,6 +98,7 @@ object ForestChangeDiagnosticDF extends SummaryDF { "tree_cover_loss_by_country_yearly", "tree_cover_loss_by_country_wdpa_yearly", "tree_cover_loss_by_country_landmark_yearly", + "tree_cover_loss_by_country_classified_region_yearly", "tree_cover_loss_arg_otbn_yearly", // treeCoverLossARGOTBNYearly "tree_cover_loss_sea_landcover_yearly", // treeCoverLossSEAsiaLandCoverYearly "tree_cover_loss_idn_landcover_yearly", // treeCoverLossIDNLandCoverYearly @@ -107,10 +108,12 @@ object ForestChangeDiagnosticDF extends SummaryDF { "tree_cover_loss_prodes_yearly", // prodesLossYearly "tree_cover_loss_prodes_wdpa_yearly", // prodesLossProtectedAreasYearly "tree_cover_loss_prodes_primary_forest_yearly", // prodesLossProdesPrimaryForestYearly + "country_area", "country_specific_deforestation_yearly", "country_specific_deforestation_wdpa_yearly", "country_specific_deforestation_landmark_yearly", "country_specific_deforestation_classified_region_yearly", + "classified_region_area", "tree_cover_loss_brazil_biomes_yearly", // treeCoverLossBRABiomesYearly "tree_cover_extent_total", // treeCoverExtent "tree_cover_extent_primary_forest", // treeCoverExtentPrimaryForest diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala index 1a01b0c2..20b943c7 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala @@ -18,11 +18,14 @@ case class ForestChangeDiagnosticData( tree_cover_loss_intact_forest_yearly: ForestChangeDiagnosticDataLossYearly, tree_cover_loss_protected_areas_yearly: ForestChangeDiagnosticDataLossYearly, // The "tree_cover_loss_by_country_*" fields are only filled in per-country when - // countryCode != "" (currently only ARG and BRA). These are for umdLoss + // countryCode != "" (currently only ARG, BRA, and COL). These are for umdLoss // comparison bars within the country-specific UI tabs. tree_cover_loss_by_country_yearly: ForestChangeDiagnosticDataLossYearlyCategory, tree_cover_loss_by_country_wdpa_yearly: ForestChangeDiagnosticDataLossYearlyTwoCategory, tree_cover_loss_by_country_landmark_yearly: ForestChangeDiagnosticDataLossYearlyTwoCategory, + // Tree cover loss by country-specified classifications. For ARG, it is breakdown + // by OTBN categories, and for COL, it is breakdown by Frontera Agricola categories). + tree_cover_loss_by_country_classified_region_yearly: ForestChangeDiagnosticDataLossYearlyTwoCategory, /** Tree cover loss in Argentina Native Forest Land Plan (OTBN) categories */ tree_cover_loss_arg_otbn_yearly: ForestChangeDiagnosticDataLossYearlyCategory, /** Tree cover loss in south east asia */ @@ -37,15 +40,21 @@ case class ForestChangeDiagnosticData( /** prodesLossProtectedAreasYearly */ tree_cover_loss_prodes_wdpa_yearly: ForestChangeDiagnosticDataLossYearly, tree_cover_loss_prodes_primary_forest_yearly: ForestChangeDiagnosticDataLossYearly, + // The area in each of ARG, BRA, and COL. Only has entries for countries with + // non-zero overlap. + country_area: ForestChangeDiagnosticDataDoubleCategory, // The "country_specific_deforestation_*" fields use the country-specific - // deforestation data, which may have approximate years (because some - // deforestation is only specified for a range of years) + // deforestation data (currently only ARG and BRA), which may have approximate + // years (because some deforestation is only specified for a range of years) country_specific_deforestation_yearly: ForestChangeDiagnosticDataLossApproxYearlyCategory, country_specific_deforestation_wdpa_yearly: ForestChangeDiagnosticDataLossApproxYearlyTwoCategory, country_specific_deforestation_landmark_yearly: ForestChangeDiagnosticDataLossApproxYearlyTwoCategory, // This is a country-specific categorization of country-specific deforestation. For - // Argentina, it is a breakdown into OTBN categories. + // ARG, it is a breakdown by OTBN categories. country_specific_deforestation_classified_region_yearly: ForestChangeDiagnosticDataLossApproxYearlyTwoCategory, + // Breakdown by country and country-specific category (OTBN for ARG and Frontera + // Agricola for COL) + classified_region_area: ForestChangeDiagnosticDataDoubleTwoCategory, tree_cover_loss_brazil_biomes_yearly: ForestChangeDiagnosticDataLossYearlyCategory, tree_cover_extent_total: ForestChangeDiagnosticDataDouble, tree_cover_extent_primary_forest: ForestChangeDiagnosticDataDouble, @@ -126,6 +135,9 @@ case class ForestChangeDiagnosticData( tree_cover_loss_by_country_landmark_yearly.merge( other.tree_cover_loss_by_country_landmark_yearly ), + tree_cover_loss_by_country_classified_region_yearly.merge( + other.tree_cover_loss_by_country_classified_region_yearly + ), tree_cover_loss_arg_otbn_yearly.merge( other.tree_cover_loss_arg_otbn_yearly ), @@ -151,10 +163,12 @@ case class ForestChangeDiagnosticData( tree_cover_loss_prodes_primary_forest_yearly.merge( other.tree_cover_loss_prodes_primary_forest_yearly ), + country_area.merge(other.country_area), country_specific_deforestation_yearly.merge(other.country_specific_deforestation_yearly), country_specific_deforestation_wdpa_yearly.merge(other.country_specific_deforestation_wdpa_yearly), country_specific_deforestation_landmark_yearly.merge(other.country_specific_deforestation_landmark_yearly), country_specific_deforestation_classified_region_yearly.merge(other.country_specific_deforestation_classified_region_yearly), + classified_region_area.merge(other.classified_region_area), tree_cover_loss_brazil_biomes_yearly.merge(other.tree_cover_loss_brazil_biomes_yearly), tree_cover_extent_total.merge(other.tree_cover_extent_total), tree_cover_extent_primary_forest.merge(other.tree_cover_extent_primary_forest), @@ -322,6 +336,7 @@ object ForestChangeDiagnosticData { ForestChangeDiagnosticDataLossYearlyCategory.empty, ForestChangeDiagnosticDataLossYearlyTwoCategory.empty, ForestChangeDiagnosticDataLossYearlyTwoCategory.empty, + ForestChangeDiagnosticDataLossYearlyTwoCategory.empty, ForestChangeDiagnosticDataLossYearlyCategory.empty, ForestChangeDiagnosticDataLossYearlyCategory.empty, ForestChangeDiagnosticDataLossYearlyCategory.empty, @@ -331,10 +346,12 @@ object ForestChangeDiagnosticData { ForestChangeDiagnosticDataLossYearly.empty, ForestChangeDiagnosticDataLossYearly.empty, ForestChangeDiagnosticDataLossYearly.empty, + ForestChangeDiagnosticDataDoubleCategory.empty, ForestChangeDiagnosticDataLossApproxYearlyCategory.empty, ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.empty, ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.empty, ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.empty, + ForestChangeDiagnosticDataDoubleTwoCategory.empty, ForestChangeDiagnosticDataLossYearlyCategory.empty, ForestChangeDiagnosticDataDouble.empty, ForestChangeDiagnosticDataDouble.empty, diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleCategory.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleCategory.scala index 3ae5cf5c..2f5862b5 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleCategory.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleCategory.scala @@ -19,14 +19,14 @@ case class ForestChangeDiagnosticDataDoubleCategory( }) } + def round: Map[String, Double] = + this.value.map { + case (key, value) => + key -> value.round + } + def toJson: String = { - this.value - .map { - case (key, value) => - key -> value.round - } - .asJson - .noSpaces + this.round.asJson.noSpaces } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleTwoCategory.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleTwoCategory.scala new file mode 100644 index 00000000..5306e21b --- /dev/null +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleTwoCategory.scala @@ -0,0 +1,62 @@ +package org.globalforestwatch.summarystats.forest_change_diagnostic + +import frameless.Injection +import io.circe.syntax._ +import io.circe.parser.decode + +case class ForestChangeDiagnosticDataDoubleTwoCategory( + value: Map[String, ForestChangeDiagnosticDataDoubleCategory] + ) extends ForestChangeDiagnosticDataParser[ForestChangeDiagnosticDataDoubleTwoCategory] { + def merge( + other: ForestChangeDiagnosticDataDoubleTwoCategory + ): ForestChangeDiagnosticDataDoubleTwoCategory = { + + ForestChangeDiagnosticDataDoubleTwoCategory(value ++ other.value.map { + case (key, otherValue) => + key -> value + .getOrElse(key, ForestChangeDiagnosticDataDoubleCategory.empty) + .merge(otherValue) + }) + } + + def toJson: String = { + this.value + .map { + case (key, value) => + key -> value.round + } + .asJson + .noSpaces + } +} + +object ForestChangeDiagnosticDataDoubleTwoCategory { + def empty: ForestChangeDiagnosticDataDoubleTwoCategory = + ForestChangeDiagnosticDataDoubleTwoCategory(Map()) + + def fill( + className1: String, + className2: String, + areaHa: Double, + noData: List[String] = List("", "Unknown", "Not applicable"), + include: Boolean = true + ): ForestChangeDiagnosticDataDoubleTwoCategory = { + if (noData.contains(className1)) + ForestChangeDiagnosticDataDoubleTwoCategory.empty + else + ForestChangeDiagnosticDataDoubleTwoCategory( + Map(className1 -> ForestChangeDiagnosticDataDoubleCategory.fill(className2, areaHa, include = include)) + ) + } + + def fromString(value: String): ForestChangeDiagnosticDataDoubleTwoCategory = { + + val categories: Map[String, String] = decode[Map[String, String]](value).getOrElse(Map()) + val newValue: Map[String, ForestChangeDiagnosticDataDoubleCategory] = categories.map { case (k, v) => (k, ForestChangeDiagnosticDataDoubleCategory.fromString(v)) } + ForestChangeDiagnosticDataDoubleTwoCategory(newValue) + + } + + implicit def injection: Injection[ForestChangeDiagnosticDataDoubleTwoCategory , String] = Injection(_.toJson, fromString) +} + diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossApproxYearlyCategory.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossApproxYearlyCategory.scala index 663d2630..e634a875 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossApproxYearlyCategory.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossApproxYearlyCategory.scala @@ -52,7 +52,7 @@ object ForestChangeDiagnosticDataLossApproxYearlyCategory { include: Boolean = true ): ForestChangeDiagnosticDataLossApproxYearlyCategory = { - if (noData.contains(className)) + if (noData.contains(className) || !include) ForestChangeDiagnosticDataLossApproxYearlyCategory.empty else ForestChangeDiagnosticDataLossApproxYearlyCategory( diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.scala index 5a667721..97468b6d 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.scala @@ -52,7 +52,7 @@ object ForestChangeDiagnosticDataLossApproxYearlyTwoCategory { include: Boolean = true ): ForestChangeDiagnosticDataLossApproxYearlyTwoCategory = { - if (noData.contains(categoryName)) + if (noData.contains(categoryName) || !include) ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.empty else ForestChangeDiagnosticDataLossApproxYearlyTwoCategory( diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyCategory.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyCategory.scala index 99d6491f..41811e1e 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyCategory.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyCategory.scala @@ -48,7 +48,7 @@ object ForestChangeDiagnosticDataLossYearlyCategory { include: Boolean = true ): ForestChangeDiagnosticDataLossYearlyCategory = { - if (noData.contains(className)) + if (noData.contains(className) || !include) ForestChangeDiagnosticDataLossYearlyCategory.empty else ForestChangeDiagnosticDataLossYearlyCategory( diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyTwoCategory.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyTwoCategory.scala index 5a1196fb..d390ed99 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyTwoCategory.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyTwoCategory.scala @@ -50,7 +50,7 @@ object ForestChangeDiagnosticDataLossYearlyTwoCategory { include: Boolean = true ): ForestChangeDiagnosticDataLossYearlyTwoCategory = { - if (noData.contains(categoryName)) + if (noData.contains(categoryName) || !include) ForestChangeDiagnosticDataLossYearlyTwoCategory.empty else ForestChangeDiagnosticDataLossYearlyTwoCategory( diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticGridSources.scala index f5c5e9c4..d0e3d1c6 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticGridSources.scala @@ -29,6 +29,7 @@ case class ForestChangeDiagnosticGridSources(gridTile: GridTile, kwargs: Map[Str val protectedAreasByCategory: DetailedProtectedAreas = DetailedProtectedAreas(gridTile, kwargs) val landmark: Landmark = Landmark(gridTile, kwargs) val argForestLoss: ArgForestLoss = ArgForestLoss(gridTile, kwargs) + val colFronteraAgricola: ColFronteraAgricola = ColFronteraAgricola(gridTile, kwargs) def readWindow( windowKey: SpatialKey, diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawDataGroup.scala index bb874f1f..ef28551f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawDataGroup.scala @@ -32,6 +32,7 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int, argPresence: Boolean, protectedAreaByCategory: String, landmarkByCategory: String, + classifiedRegion: String, ) { /** Produce a partial ForestChangeDiagnosticData only for the loss year in this data group */ @@ -83,7 +84,7 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int, protectedAreaByCategory, umdTreeCoverLossYear, totalArea, - include = isProtectedArea && isUMDLoss + include = protectedAreaByCategory != "" && isUMDLoss ), tree_cover_loss_by_country_landmark_yearly = ForestChangeDiagnosticDataLossYearlyTwoCategory.fill( @@ -93,6 +94,14 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int, totalArea, include = landmarkByCategory != "" && isUMDLoss ), + tree_cover_loss_by_country_classified_region_yearly = + ForestChangeDiagnosticDataLossYearlyTwoCategory.fill( + countryCode, + classifiedRegion, + umdTreeCoverLossYear, + totalArea, + include = classifiedRegion != "" && isUMDLoss + ), tree_cover_loss_arg_otbn_yearly = ForestChangeDiagnosticDataLossYearlyCategory.fill( argOTBN, @@ -150,6 +159,8 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int, totalArea, isProdesLoss && isPrimaryForest ), + country_area = ForestChangeDiagnosticDataDoubleCategory + .fill(countryCode, totalArea), country_specific_deforestation_yearly = ForestChangeDiagnosticDataLossApproxYearlyCategory.fill( countryCode, countrySpecificLossYear, @@ -182,6 +193,17 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int, } else { ForestChangeDiagnosticDataLossApproxYearlyTwoCategory.empty, }, + classified_region_area = + if (countryCode =="ARG" || countryCode == "COL") { + ForestChangeDiagnosticDataDoubleTwoCategory.fill( + countryCode, + classifiedRegion, + totalArea, + include = classifiedRegion != "" + ) + } else { + ForestChangeDiagnosticDataDoubleTwoCategory.empty + }, tree_cover_loss_brazil_biomes_yearly = ForestChangeDiagnosticDataLossYearlyCategory.fill( braBiomes, diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticSummary.scala index d01c7774..48d56cf0 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticSummary.scala @@ -6,6 +6,7 @@ import geotrellis.raster.summary.GridVisitor import org.globalforestwatch.summarystats.Summary import org.globalforestwatch.util.Geodesy import org.globalforestwatch.layers.ApproxYear +import org.globalforestwatch.layers.GFWProCoverage /** LossData Summary by year */ case class ForestChangeDiagnosticSummary( @@ -88,10 +89,11 @@ object ForestChangeDiagnosticSummary { } } - val gfwProCoverage: Map[String, Boolean] = - raster.tile.gfwProCoverage.getData(col, row) - val argPresence = gfwProCoverage.getOrElse("Argentina", false) - val braBiomesPresence = gfwProCoverage.getOrElse("Brazil Biomes", false) + val region: Int = raster.tile.gfwProCoverage.getData(col, row) + val argPresence = GFWProCoverage.isArgentina(region) + val colPresence = GFWProCoverage.isColombia(region) + val braBiomesPresence = GFWProCoverage.isBrazilBiomesPresence(region) + val argOTBN: String = raster.tile.argOTBN.getData(col, row) // We compute country-specific forest loss using argForestLoss tile for // Argentina, and prodesLossYear for Brazil. In the very unusual case where a @@ -102,8 +104,14 @@ object ForestChangeDiagnosticSummary { // more than one country. var countrySpecificLossYear = ApproxYear(0, false) var countryCode = "" - if (argPresence) { + var classifiedRegion = "" + + if (colPresence) { + countryCode = "COL" + classifiedRegion = raster.tile.colFronteraAgricola.getData(col, row) + } else if (argPresence) { countrySpecificLossYear = raster.tile.argForestLoss.getData(col, row) + classifiedRegion = argOTBN countryCode = "ARG" } else { val possLoss = raster.tile.prodesLossYear.getData(col, row) @@ -129,7 +137,6 @@ object ForestChangeDiagnosticSummary { raster.tile.isIDNForestMoratorium.getData(col, row) val braBiomes: String = raster.tile.braBiomes.getData(col, row) val isPlantation: Boolean = raster.tile.isPlantation.getData(col, row) - val argOTBN: String = raster.tile.argOTBN.getData(col, row) // compute Booleans val isTreeCoverExtent30: Boolean = tcd2000 > 30 @@ -138,31 +145,28 @@ object ForestChangeDiagnosticSummary { val isProdesLoss: Boolean = prodesLossYear > 0 val isCountrySpecificLoss: Boolean = countrySpecificLossYear.year > 0 - val southAmericaPresence = - gfwProCoverage.getOrElse("South America", false) - val legalAmazonPresence = - gfwProCoverage.getOrElse("Legal Amazon", false) - val cerradoBiomesPresence = - gfwProCoverage.getOrElse("Cerrado Biomes", false) - val seAsiaPresence = gfwProCoverage.getOrElse("South East Asia", false) - val idnPresence = gfwProCoverage.getOrElse("Indonesia", false) + val southAmericaPresence = GFWProCoverage.isSouthAmerica(region) + val legalAmazonPresence = GFWProCoverage.isLegalAmazonPresence(region) + val cerradoBiomesPresence = GFWProCoverage.isCerradoBiomesPresence(region) + val seAsiaPresence = GFWProCoverage.isSouthEastAsia(region) + val idnPresence = GFWProCoverage.isIndonesia(region) val protectedAreaCategory = raster.tile.protectedAreasByCategory.getData(col, row) val isProtectedArea = (protectedAreaCategory != "") // Currently, only do the area intersection with the detailed WDPA categories - // if location is in Argentina. Similarly, only do area intersection with - // Landmark (indigenous territories) if in Argentina. + // if location is in ARG or COL. Similarly, only do area intersection with + // Landmark (indigenous territories) if in ARG or COL. // With lazy tile loading, the landmark tiles are only loaded if - // argPresence is true. - val detailedWdpa = if (argPresence) + // argPresence or colPresence is true. + val detailedWdpa = if (argPresence || colPresence) protectedAreaCategory else "" // We will likely have different Landmark categories for other countries, but // there is no distinction currently for Argentina, so we put all of the // indigenous area into the "Not Reported" category. - val landmarkCategory = if (argPresence) + val landmarkCategory = if (argPresence || colPresence) (if (raster.tile.landmark.getData(col, row)) "Not Reported" else "") else "" @@ -198,6 +202,7 @@ object ForestChangeDiagnosticSummary { argPresence, detailedWdpa, landmarkCategory, + classifiedRegion, ) val summaryData: ForestChangeDiagnosticRawData = diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticTile.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticTile.scala index 7c9c52e7..3eacea01 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticTile.scala @@ -34,6 +34,7 @@ case class ForestChangeDiagnosticTile( lazy val protectedAreasByCategory = sources.protectedAreasByCategory.fetchWindow(windowKey, windowLayout) lazy val landmark = sources.landmark.fetchWindow(windowKey, windowLayout) lazy val argForestLoss = sources.argForestLoss.fetchWindow(windowKey, windowLayout) + lazy val colFronteraAgricola = sources.colFronteraAgricola.fetchWindow(windowKey, windowLayout) def cellType: CellType = loss.cellType diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/package.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/package.scala index ef1c370f..03dd3b61 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/package.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/package.scala @@ -26,4 +26,7 @@ package object forest_change_diagnostic { implicit def dataLossYearlyTwoCategoryTypedEncoder: TypedEncoder[ForestChangeDiagnosticDataLossYearlyTwoCategory] = TypedEncoder.usingInjection[ForestChangeDiagnosticDataLossYearlyTwoCategory, String] + implicit def dataDoubleTwoCategoryTypedEncoder: TypedEncoder[ForestChangeDiagnosticDataDoubleTwoCategory] = + TypedEncoder.usingInjection[ForestChangeDiagnosticDataDoubleTwoCategory, String] + } diff --git a/src/test/resources/argbra-fcd-output/part-00000-3d339776-5c3b-4db8-8cac-3d284de2cca7-c000.csv b/src/test/resources/argbra-fcd-output/part-00000-3d339776-5c3b-4db8-8cac-3d284de2cca7-c000.csv new file mode 100644 index 00000000..b9994490 --- /dev/null +++ b/src/test/resources/argbra-fcd-output/part-00000-3d339776-5c3b-4db8-8cac-3d284de2cca7-c000.csv @@ -0,0 +1,2 @@ +list_id location_id status_code location_error tree_cover_loss_total_yearly tree_cover_loss_primary_forest_yearly tree_cover_loss_peat_yearly tree_cover_loss_intact_forest_yearly tree_cover_loss_protected_areas_yearly tree_cover_loss_by_country_yearly tree_cover_loss_by_country_wdpa_yearly tree_cover_loss_by_country_landmark_yearly tree_cover_loss_by_country_classified_region_yearly tree_cover_loss_arg_otbn_yearly tree_cover_loss_sea_landcover_yearly tree_cover_loss_idn_landcover_yearly tree_cover_loss_soy_yearly tree_cover_loss_idn_legal_yearly tree_cover_loss_idn_forest_moratorium_yearly tree_cover_loss_prodes_yearly tree_cover_loss_prodes_wdpa_yearly tree_cover_loss_prodes_primary_forest_yearly country_area country_specific_deforestation_yearly country_specific_deforestation_wdpa_yearly country_specific_deforestation_landmark_yearly country_specific_deforestation_classified_region_yearly classified_region_area tree_cover_loss_brazil_biomes_yearly tree_cover_extent_total tree_cover_extent_primary_forest tree_cover_extent_protected_areas tree_cover_extent_peat tree_cover_extent_intact_forest natural_habitat_primary natural_habitat_intact_forest total_area protected_areas_area peat_area arg_otbn_area protected_areas_by_category_area landmark_by_category_area brazil_biomes idn_legal_area sea_landcover_area idn_landcover_area idn_forest_moratorium_area south_america_presence legal_amazon_presence brazil_biomes_presence cerrado_biome_presence southeast_asia_presence indonesia_presence argentina_presence commodity_value_forest_extent commodity_value_peat commodity_value_protected_areas commodity_threat_deforestation commodity_threat_peat commodity_threat_protected_areas commodity_threat_fires +166 1 2 {"2001":390.3801,"2002":337.1303,"2003":591.747,"2004":522.3103,"2005":905.0571,"2006":861.6216,"2007":688.9958,"2008":719.6228,"2009":682.3401,"2010":342.0608,"2011":832.8575,"2012":822.2161,"2013":698.7244,"2014":1061.5944,"2015":623.7254,"2016":1506.2833,"2017":2137.3785,"2018":999.0414,"2019":1187.6768,"2020":1380.8694,"2021":1630.7097,"2022":1173.8577,"2023":935.707} {"2001":4.4721,"2002":4.6137,"2003":6.8156,"2004":2.1372,"2005":6.1991,"2006":69.8527,"2007":54.001,"2008":103.172,"2009":68.8725,"2010":25.6512,"2011":40.7394,"2012":61.5105,"2013":77.835,"2014":58.1765,"2015":40.3091,"2016":68.4074,"2017":163.7431,"2018":35.5811,"2019":48.25,"2020":30.8854,"2021":33.2186,"2022":47.9914,"2023":34.3442} {"2001":0.4816,"2002":0.2062,"2003":0.2062,"2004":0.6189,"2005":0.6193,"2006":0.413,"2007":1.3096,"2008":0.3435,"2009":0.6872,"2010":0.619,"2011":2.2689,"2012":1.4448,"2013":1.8558,"2014":5.7042,"2015":1.6496,"2016":2.7495,"2017":11.0043,"2018":2.4757,"2019":3.92,"2020":3.3036,"2021":5.0209,"2022":2.4071,"2023":1.7176} {} {"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6777,"2012":35.612,"2013":11.0741,"2014":20.1668,"2015":18.4411,"2016":11.7733,"2017":36.2816,"2018":7.9929,"2019":22.3112,"2020":8.1953,"2021":25.8224,"2022":11.5036,"2023":0.4131} {"ARG":{"2001":169.7492,"2002":110.0137,"2003":279.3034,"2004":243.8448,"2005":503.571,"2006":590.8377,"2007":450.6176,"2008":539.3522,"2009":450.1686,"2010":266.6501,"2011":375.4212,"2012":535.6973,"2013":450.3713,"2014":590.4731,"2015":350.2188,"2016":606.0677,"2017":909.5434,"2018":472.945,"2019":546.4409,"2020":656.0203,"2021":878.4245,"2022":433.7453,"2023":356.2457},"BRA":{"2001":220.6309,"2002":227.1166,"2003":312.4436,"2004":278.4655,"2005":401.4861,"2006":270.7838,"2007":237.4137,"2008":180.2706,"2009":232.1715,"2010":75.4107,"2011":457.3676,"2012":286.5188,"2013":248.2153,"2014":470.7769,"2015":273.5065,"2016":899.9401,"2017":1222.5303,"2018":526.0963,"2019":639.9957,"2020":724.7802,"2021":751.8719,"2022":739.5609,"2023":579.4613}} {"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6089,"2012":35.5432,"2013":10.9364,"2014":20.0291,"2015":18.4411,"2016":11.5667,"2017":36.1438,"2018":7.9929,"2019":22.3112,"2020":8.1264,"2021":25.409,"2022":11.5036,"2023":0.4131}}} {"ARG":{"Not Reported":{"2001":39.904,"2002":36.1895,"2003":118.8314,"2004":99.1471,"2005":211.2132,"2006":230.4318,"2007":162.2055,"2008":142.2109,"2009":130.0801,"2010":68.1056,"2011":108.2244,"2012":241.5664,"2013":127.9175,"2014":210.2876,"2015":139.705,"2016":247.6686,"2017":361.3402,"2018":201.2981,"2019":248.205,"2020":277.617,"2021":433.2285,"2022":149.1649,"2023":117.5902}}} {"ARG":{"Category I":{"2001":1.0996,"2002":0.8246,"2003":2.1303,"2004":0.0,"2005":0.9622,"2006":0.55,"2007":0.0,"2008":0.3437,"2009":3.7798,"2010":0.0,"2011":0.5498,"2012":0.4125,"2013":1.6495,"2014":1.5121,"2015":6.3928,"2016":1.6495,"2017":2.0618,"2018":0.1375,"2019":0.6873,"2020":1.5121,"2021":2.8869,"2022":0.2064,"2023":0.0},"Category II":{"2001":15.5017,"2002":13.3015,"2003":31.5663,"2004":34.1258,"2005":67.4062,"2006":55.745,"2007":46.5984,"2008":73.8253,"2009":131.6487,"2010":73.2082,"2011":100.0926,"2012":133.2506,"2013":130.0616,"2014":156.9412,"2015":106.1384,"2016":151.6383,"2017":269.1299,"2018":137.5692,"2019":158.3015,"2020":163.4794,"2021":243.9769,"2022":130.3351,"2023":128.5559},"Category III":{"2001":26.2709,"2002":13.8555,"2003":42.6008,"2004":23.2359,"2005":54.4591,"2006":44.0493,"2007":27.9229,"2008":52.6735,"2009":78.9423,"2010":73.1421,"2011":68.5956,"2012":112.4446,"2013":151.5104,"2014":217.1291,"2015":113.4622,"2016":201.8612,"2017":294.2968,"2018":150.2793,"2019":194.1712,"2020":223.0738,"2021":300.5095,"2022":110.6482,"2023":98.4466}}} {"Category I":{"2001":1.0996,"2002":0.8246,"2003":2.1303,"2004":0.0,"2005":0.9622,"2006":0.55,"2007":0.0,"2008":0.3437,"2009":3.7798,"2010":0.0,"2011":0.5498,"2012":0.4125,"2013":1.6495,"2014":1.5121,"2015":6.3928,"2016":1.6495,"2017":2.0618,"2018":0.1375,"2019":0.6873,"2020":1.5121,"2021":2.8869,"2022":0.2064,"2023":0.0},"Category II":{"2001":15.5017,"2002":13.3015,"2003":31.5663,"2004":34.1258,"2005":67.4062,"2006":55.745,"2007":46.5984,"2008":73.8253,"2009":131.6487,"2010":73.2082,"2011":100.0926,"2012":133.2506,"2013":130.0616,"2014":157.0101,"2015":106.1384,"2016":151.6383,"2017":269.1299,"2018":137.5692,"2019":158.3015,"2020":163.4794,"2021":244.0458,"2022":130.3351,"2023":128.5559},"Category III":{"2001":26.2709,"2002":13.8555,"2003":42.6008,"2004":23.2359,"2005":54.4591,"2006":44.0493,"2007":27.9229,"2008":52.6735,"2009":78.9423,"2010":73.1421,"2011":68.5956,"2012":112.4446,"2013":151.5104,"2014":217.1291,"2015":113.4622,"2016":201.8612,"2017":294.2968,"2018":150.2793,"2019":194.1712,"2020":223.0738,"2021":300.5095,"2022":110.6482,"2023":98.4466}} {} {} {"2001":21.624,"2002":20.8704,"2003":35.0478,"2004":35.6016,"2005":35.1084,"2006":29.4707,"2007":23.7689,"2008":26.0851,"2009":23.4664,"2010":9.7672,"2011":76.1005,"2012":28.4983,"2013":18.1678,"2014":49.4618,"2015":21.3388,"2016":63.0798,"2017":82.2563,"2018":30.9672,"2019":32.1454,"2020":38.9536,"2021":3.9932,"2022":1.102} {} {} {} {} {} {"BRA":221184.6524,"ARG":194985.6634} {"ARG":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":186.9026,"2008":0.0,"2009":0.0,"2010approx":417.5112,"2011":15.1053,"2012":0.0,"2013approx":117.2137,"2014":79.0631,"2015":3.5847,"2016":8.0709,"2017":163.5411,"2018":31.4482,"2019":47.2747,"2020":181.2011,"2021":110.5264,"2022":18.9678,"2023":0.0}} {"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010approx":1.0342,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":17.5539,"2020":23.7832,"2021":0.0,"2022":0.0,"2023":0.0}}} {"ARG":{"Not Reported":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":92.4229,"2008":0.0,"2009":0.0,"2010approx":46.7539,"2011":0.0,"2012":0.0,"2013approx":20.4653,"2014":46.5229,"2015":0.0,"2016":0.0,"2017":44.9934,"2018":5.5842,"2019":31.0673,"2020":76.3528,"2021":21.096,"2022":0.0,"2023":0.0}}} {"ARG":{"Category I":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":21.3774,"2021":0.0,"2022":0.0,"2023":0.0},"Category III":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":14.683,"2008":0.0,"2009":0.0,"2010approx":75.2363,"2011":12.4841,"2012":0.0,"2013approx":30.8262,"2014":39.6991,"2015":0.0,"2016":6.8293,"2017":94.9476,"2018":22.8267,"2019":13.5189,"2020":40.3374,"2021":34.8144,"2022":10.8284,"2023":0.0},"Category II":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":25.0997,"2008":0.0,"2009":0.0,"2010approx":174.5175,"2011":1.7935,"2012":0.0,"2013approx":71.4984,"2014":27.7832,"2015":3.5847,"2016":1.2417,"2017":54.5984,"2018":8.3456,"2019":32.4472,"2020":73.3842,"2021":66.2684,"2022":7.1053,"2023":0.0}}} {"ARG":{"Category II":122991.3614,"Category I":30089.6869,"Category III":11275.7574}} {"Mata Atlântica":{"2001":222.281,"2002":229.664,"2003":313.889,"2004":281.566,"2005":404.1011,"2006":274.0218,"2007":244.2288,"2008":182.4026,"2009":237.3952,"2010":77.4035,"2011":460.1882,"2012":303.0526,"2013":257.9101,"2014":491.4761,"2015":279.4206,"2016":909.6394,"2017":1238.9614,"2018":529.7395,"2019":649.9751,"2020":733.7938,"2021":765.01,"2022":743.0739,"2023":584.0024}} 310497.0062 151998.5232 145836.3381 3133.6034 0.0 152001.8248 0.0 416412.5376 145907.1635 3633.9932 {"Category II":123789.927,"Category I":30089.6869,"Category III":11275.7574} {"UNESCO-MAB Biosphere Reserve":144721.3787} {"Not Reported":91313.3874} {"Mata Atlântica":235455.6349} {} {} {} 0.0 true false true false false false true {"2001":228399.0302,"2002":228399.0302,"2003":228218.695,"2004":228019.807,"2005":227669.5127,"2006":227360.6322,"2007":226793.1945,"2008":226294.0263,"2009":225908.1898,"2010":225416.7831,"2011":224990.0743,"2012":224720.0969,"2013":224202.2054,"2014":223703.987,"2015":223262.4402,"2016":222610.9217,"2017":222244.0711,"2018":221464.0632,"2019":220326.1549,"2020":219785.4113,"2021":219159.8818,"2022":218487.8012} {"2001":3633.9932,"2002":3633.9932,"2003":3633.9932,"2004":3633.9932,"2005":3633.9932,"2006":3633.9932,"2007":3633.9932,"2008":3633.9932,"2009":3633.9932,"2010":3633.9932,"2011":3633.9932,"2012":3633.9932,"2013":3633.9932,"2014":3633.9932,"2015":3633.9932,"2016":3633.9932,"2017":3633.9932,"2018":3633.9932,"2019":3633.9932,"2020":3633.9932,"2021":3633.9932,"2022":3633.9932} {"2001":145907.1635,"2002":145907.1635,"2003":145907.1635,"2004":145907.1635,"2005":145907.1635,"2006":145907.1635,"2007":145907.1635,"2008":145907.1635,"2009":145907.1635,"2010":145907.1635,"2011":145907.1635,"2012":145907.1635,"2013":145907.1635,"2014":145907.1635,"2015":145907.1635,"2016":145907.1635,"2017":145907.1635,"2018":145907.1635,"2019":145907.1635,"2020":145907.1635,"2021":145907.1635,"2022":145907.1635} {"2002":379.2232,"2003":549.1823,"2004":659.1748,"2005":876.3182,"2006":1066.606,"2007":885.0047,"2008":877.2432,"2009":918.1155,"2010":696.6862,"2011":787.8689,"2012":1016.1099,"2013":939.7652,"2014":1093.0653,"2015":1018.3691,"2016":1146.8585,"2017":1917.9162,"2018":1678.6519,"2019":1166.2731,"2020":1297.6101,"2021":1606.9852,"2022":1513.9363} {"2002":20.9771,"2003":20.5645,"2004":20.8397,"2005":21.1152,"2006":21.1155,"2007":21.9433,"2008":21.8738,"2009":21.0455,"2010":21.321,"2011":21.8711,"2012":22.0087,"2013":21.9396,"2014":22.4891,"2015":22.2141,"2016":22.4208,"2017":26.4784,"2018":25.7222,"2019":22.0779,"2020":22.2157,"2021":23.3861,"2022":23.3168} {"2002":1412.3095,"2003":1411.9654,"2004":1409.5569,"2005":1412.3853,"2006":1422.5074,"2007":1433.1795,"2008":1430.6331,"2009":1426.769,"2010":1421.6073,"2011":1416.8048,"2012":1421.3469,"2013":1419.7406,"2014":1420.2105,"2015":1426.2686,"2016":1424.1418,"2017":1427.8639,"2018":1422.7713,"2019":1422.0236,"2020":1426.6357,"2021":1427.8733,"2022":1431.9397} {} diff --git a/src/test/resources/argbra-fcd-output/part-00000-95345206-7d33-484d-bead-858f0b3ea115-c000.csv b/src/test/resources/argbra-fcd-output/part-00000-95345206-7d33-484d-bead-858f0b3ea115-c000.csv deleted file mode 100644 index ef94ab54..00000000 --- a/src/test/resources/argbra-fcd-output/part-00000-95345206-7d33-484d-bead-858f0b3ea115-c000.csv +++ /dev/null @@ -1,2 +0,0 @@ -list_id location_id status_code location_error tree_cover_loss_total_yearly tree_cover_loss_primary_forest_yearly tree_cover_loss_peat_yearly tree_cover_loss_intact_forest_yearly tree_cover_loss_protected_areas_yearly tree_cover_loss_by_country_yearly tree_cover_loss_by_country_wdpa_yearly tree_cover_loss_by_country_landmark_yearly tree_cover_loss_arg_otbn_yearly tree_cover_loss_sea_landcover_yearly tree_cover_loss_idn_landcover_yearly tree_cover_loss_soy_yearly tree_cover_loss_idn_legal_yearly tree_cover_loss_idn_forest_moratorium_yearly tree_cover_loss_prodes_yearly tree_cover_loss_prodes_wdpa_yearly tree_cover_loss_prodes_primary_forest_yearly country_specific_deforestation_yearly country_specific_deforestation_wdpa_yearly country_specific_deforestation_landmark_yearly country_specific_deforestation_classified_region_yearly tree_cover_loss_brazil_biomes_yearly tree_cover_extent_total tree_cover_extent_primary_forest tree_cover_extent_protected_areas tree_cover_extent_peat tree_cover_extent_intact_forest natural_habitat_primary natural_habitat_intact_forest total_area protected_areas_area peat_area arg_otbn_area protected_areas_by_category_area landmark_by_category_area brazil_biomes idn_legal_area sea_landcover_area idn_landcover_area idn_forest_moratorium_area south_america_presence legal_amazon_presence brazil_biomes_presence cerrado_biome_presence southeast_asia_presence indonesia_presence argentina_presence commodity_value_forest_extent commodity_value_peat commodity_value_protected_areas commodity_threat_deforestation commodity_threat_peat commodity_threat_protected_areas commodity_threat_fires -166 1 2 {"2001":390.3801,"2002":337.1303,"2003":591.747,"2004":522.3103,"2005":905.0571,"2006":861.6216,"2007":688.9958,"2008":719.6228,"2009":682.3401,"2010":342.0608,"2011":832.8575,"2012":822.2161,"2013":698.7244,"2014":1061.5944,"2015":623.7254,"2016":1506.2833,"2017":2137.3785,"2018":999.0414,"2019":1187.6768,"2020":1380.8694,"2021":1630.7097,"2022":1173.8577,"2023":935.707} {"2001":4.4721,"2002":4.6137,"2003":6.8156,"2004":2.1372,"2005":6.1991,"2006":69.8527,"2007":54.001,"2008":103.172,"2009":68.8725,"2010":25.6512,"2011":40.7394,"2012":61.5105,"2013":77.835,"2014":58.1765,"2015":40.3091,"2016":68.4074,"2017":163.7431,"2018":35.5811,"2019":48.25,"2020":30.8854,"2021":33.2186,"2022":47.9914,"2023":34.3442} {"2001":0.4816,"2002":0.2062,"2003":0.2062,"2004":0.6189,"2005":0.6193,"2006":0.413,"2007":1.3096,"2008":0.3435,"2009":0.6872,"2010":0.619,"2011":2.2689,"2012":1.4448,"2013":1.8558,"2014":5.7042,"2015":1.6496,"2016":2.7495,"2017":11.0043,"2018":2.4757,"2019":3.92,"2020":3.3036,"2021":5.0209,"2022":2.4071,"2023":1.7176} {} {"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6777,"2012":35.612,"2013":11.0741,"2014":20.1668,"2015":18.4411,"2016":11.7733,"2017":36.2816,"2018":7.9929,"2019":22.3112,"2020":8.1953,"2021":25.8224,"2022":11.5036,"2023":0.4131} {"BRA":{"2001":220.6309,"2002":227.1166,"2003":312.4436,"2004":278.4655,"2005":401.4861,"2006":270.7838,"2007":237.4137,"2008":180.2706,"2009":232.1715,"2010":75.4107,"2011":457.3676,"2012":286.5188,"2013":248.2153,"2014":470.7769,"2015":273.5065,"2016":899.9401,"2017":1222.5303,"2018":526.0963,"2019":639.9957,"2020":724.7802,"2021":751.8719,"2022":739.5609,"2023":579.4613},"ARG":{"2001":169.7492,"2002":110.0137,"2003":279.3034,"2004":243.8448,"2005":503.571,"2006":590.8377,"2007":450.6176,"2008":539.3522,"2009":450.1686,"2010":266.6501,"2011":375.4212,"2012":535.6973,"2013":450.3713,"2014":590.4731,"2015":350.2188,"2016":606.0677,"2017":909.5434,"2018":472.945,"2019":546.4409,"2020":656.0203,"2021":878.4245,"2022":433.7453,"2023":356.2457}} {"BRA":{},"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6089,"2012":35.5432,"2013":10.9364,"2014":20.0291,"2015":18.4411,"2016":11.5667,"2017":36.1438,"2018":7.9929,"2019":22.3112,"2020":8.1264,"2021":25.409,"2022":11.5036,"2023":0.4131}}} {"BRA":{},"ARG":{"Not Reported":{"2001":39.904,"2002":36.1895,"2003":118.8314,"2004":99.1471,"2005":211.2132,"2006":230.4318,"2007":162.2055,"2008":142.2109,"2009":130.0801,"2010":68.1056,"2011":108.2244,"2012":241.5664,"2013":127.9175,"2014":210.2876,"2015":139.705,"2016":247.6686,"2017":361.3402,"2018":201.2981,"2019":248.205,"2020":277.617,"2021":433.2285,"2022":149.1649,"2023":117.5902}}} {"Category II":{"2001":15.5017,"2002":13.3015,"2003":31.5663,"2004":34.1258,"2005":67.4062,"2006":55.745,"2007":46.5984,"2008":73.8253,"2009":131.6487,"2010":73.2082,"2011":100.0926,"2012":133.2506,"2013":130.0616,"2014":157.0101,"2015":106.1384,"2016":151.6383,"2017":269.1299,"2018":137.5692,"2019":158.3015,"2020":163.4794,"2021":244.0458,"2022":130.3351,"2023":128.5559},"Category I":{"2001":1.0996,"2002":0.8246,"2003":2.1303,"2004":0.0,"2005":0.9622,"2006":0.55,"2007":0.0,"2008":0.3437,"2009":3.7798,"2010":0.0,"2011":0.5498,"2012":0.4125,"2013":1.6495,"2014":1.5121,"2015":6.3928,"2016":1.6495,"2017":2.0618,"2018":0.1375,"2019":0.6873,"2020":1.5121,"2021":2.8869,"2022":0.2064,"2023":0.0},"Category III":{"2001":26.2709,"2002":13.8555,"2003":42.6008,"2004":23.2359,"2005":54.4591,"2006":44.0493,"2007":27.9229,"2008":52.6735,"2009":78.9423,"2010":73.1421,"2011":68.5956,"2012":112.4446,"2013":151.5104,"2014":217.1291,"2015":113.4622,"2016":201.8612,"2017":294.2968,"2018":150.2793,"2019":194.1712,"2020":223.0738,"2021":300.5095,"2022":110.6482,"2023":98.4466}} {} {} {"2001":21.624,"2002":20.8704,"2003":35.0478,"2004":35.6016,"2005":35.1084,"2006":29.4707,"2007":23.7689,"2008":26.0851,"2009":23.4664,"2010":9.7672,"2011":76.1005,"2012":28.4983,"2013":18.1678,"2014":49.4618,"2015":21.3388,"2016":63.0798,"2017":82.2563,"2018":30.9672,"2019":32.1454,"2020":38.9536,"2021":3.9932,"2022":1.102} {} {} {} {} {} {"BRA":{},"ARG":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":186.9026,"2008":0.0,"2009":0.0,"2010approx":417.5112,"2011":15.1053,"2012":0.0,"2013approx":117.2137,"2014":79.0631,"2015":3.5847,"2016":8.0709,"2017":163.5411,"2018":31.4482,"2019":47.2747,"2020":181.2011,"2021":110.5264,"2022":18.9678,"2023":0.0}} {"BRA":{},"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010approx":1.0342,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":17.5539,"2020":23.7832,"2021":0.0,"2022":0.0,"2023":0.0}}} {"BRA":{},"ARG":{"Not Reported":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":92.4229,"2008":0.0,"2009":0.0,"2010approx":46.7539,"2011":0.0,"2012":0.0,"2013approx":20.4653,"2014":46.5229,"2015":0.0,"2016":0.0,"2017":44.9934,"2018":5.5842,"2019":31.0673,"2020":76.3528,"2021":21.096,"2022":0.0,"2023":0.0}}} {"ARG":{"Category II":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":25.0997,"2008":0.0,"2009":0.0,"2010approx":174.5175,"2011":1.7935,"2012":0.0,"2013approx":71.4984,"2014":27.7832,"2015":3.5847,"2016":1.2417,"2017":54.5984,"2018":8.3456,"2019":32.4472,"2020":73.3842,"2021":66.2684,"2022":7.1053,"2023":0.0},"Category I":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":21.3774,"2021":0.0,"2022":0.0,"2023":0.0},"Category III":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":14.683,"2008":0.0,"2009":0.0,"2010approx":75.2363,"2011":12.4841,"2012":0.0,"2013approx":30.8262,"2014":39.6991,"2015":0.0,"2016":6.8293,"2017":94.9476,"2018":22.8267,"2019":13.5189,"2020":40.3374,"2021":34.8144,"2022":10.8284,"2023":0.0}}} {"Mata Atlântica":{"2001":222.281,"2002":229.664,"2003":313.889,"2004":281.566,"2005":404.1011,"2006":274.0218,"2007":244.2288,"2008":182.4026,"2009":237.3952,"2010":77.4035,"2011":460.1882,"2012":303.0526,"2013":257.9101,"2014":491.4761,"2015":279.4206,"2016":909.6394,"2017":1238.9614,"2018":529.7395,"2019":649.9751,"2020":733.7938,"2021":765.01,"2022":743.0739,"2023":584.0024}} 310497.0062 151998.5232 145836.3381 3133.6034 0.0 152001.8248 0.0 416412.5376 145907.1635 3633.9932 {"Category II":123789.927,"Category I":30089.6869,"Category III":11275.7574} {"UNESCO-MAB Biosphere Reserve":144721.3787} {"Not Reported":91313.3874} {"Mata Atlântica":235455.6349} {} {} {} 0.0 true false true false false false true {"2001":228399.0302,"2002":228399.0302,"2003":228218.695,"2004":228019.807,"2005":227669.5127,"2006":227360.6322,"2007":226793.1945,"2008":226294.0263,"2009":225908.1898,"2010":225416.7831,"2011":224990.0743,"2012":224720.0969,"2013":224202.2054,"2014":223703.987,"2015":223262.4402,"2016":222610.9217,"2017":222244.0711,"2018":221464.0632,"2019":220326.1549,"2020":219785.4113,"2021":219159.8818,"2022":218487.8012} {"2001":3633.9932,"2002":3633.9932,"2003":3633.9932,"2004":3633.9932,"2005":3633.9932,"2006":3633.9932,"2007":3633.9932,"2008":3633.9932,"2009":3633.9932,"2010":3633.9932,"2011":3633.9932,"2012":3633.9932,"2013":3633.9932,"2014":3633.9932,"2015":3633.9932,"2016":3633.9932,"2017":3633.9932,"2018":3633.9932,"2019":3633.9932,"2020":3633.9932,"2021":3633.9932,"2022":3633.9932} {"2001":145907.1635,"2002":145907.1635,"2003":145907.1635,"2004":145907.1635,"2005":145907.1635,"2006":145907.1635,"2007":145907.1635,"2008":145907.1635,"2009":145907.1635,"2010":145907.1635,"2011":145907.1635,"2012":145907.1635,"2013":145907.1635,"2014":145907.1635,"2015":145907.1635,"2016":145907.1635,"2017":145907.1635,"2018":145907.1635,"2019":145907.1635,"2020":145907.1635,"2021":145907.1635,"2022":145907.1635} {"2002":379.2232,"2003":549.1823,"2004":659.1748,"2005":876.3182,"2006":1066.606,"2007":885.0047,"2008":877.2432,"2009":918.1155,"2010":696.6862,"2011":787.8689,"2012":1016.1099,"2013":939.7652,"2014":1093.0653,"2015":1018.3691,"2016":1146.8585,"2017":1917.9162,"2018":1678.6519,"2019":1166.2731,"2020":1297.6101,"2021":1606.9852,"2022":1513.9363} {"2002":20.9771,"2003":20.5645,"2004":20.8397,"2005":21.1152,"2006":21.1155,"2007":21.9433,"2008":21.8738,"2009":21.0455,"2010":21.321,"2011":21.8711,"2012":22.0087,"2013":21.9396,"2014":22.4891,"2015":22.2141,"2016":22.4208,"2017":26.4784,"2018":25.7222,"2019":22.0779,"2020":22.2157,"2021":23.3861,"2022":23.3168} {"2002":1412.3095,"2003":1411.9654,"2004":1409.5569,"2005":1412.3853,"2006":1422.5074,"2007":1433.1795,"2008":1430.6331,"2009":1426.769,"2010":1421.6073,"2011":1416.8048,"2012":1421.3469,"2013":1419.7406,"2014":1420.2105,"2015":1426.2686,"2016":1424.1418,"2017":1427.8639,"2018":1422.7713,"2019":1422.0236,"2020":1426.6357,"2021":1427.8733,"2022":1431.9397} {} diff --git a/src/test/resources/col-fcd-output/part-00000-97890df8-9a99-416f-a597-d2d82cef5ee8-c000.csv b/src/test/resources/col-fcd-output/part-00000-97890df8-9a99-416f-a597-d2d82cef5ee8-c000.csv new file mode 100644 index 00000000..2b1c1e27 --- /dev/null +++ b/src/test/resources/col-fcd-output/part-00000-97890df8-9a99-416f-a597-d2d82cef5ee8-c000.csv @@ -0,0 +1,3 @@ +list_id location_id status_code location_error tree_cover_loss_total_yearly tree_cover_loss_primary_forest_yearly tree_cover_loss_peat_yearly tree_cover_loss_intact_forest_yearly tree_cover_loss_protected_areas_yearly tree_cover_loss_by_country_yearly tree_cover_loss_by_country_wdpa_yearly tree_cover_loss_by_country_landmark_yearly tree_cover_loss_by_country_classified_region_yearly tree_cover_loss_arg_otbn_yearly tree_cover_loss_sea_landcover_yearly tree_cover_loss_idn_landcover_yearly tree_cover_loss_soy_yearly tree_cover_loss_idn_legal_yearly tree_cover_loss_idn_forest_moratorium_yearly tree_cover_loss_prodes_yearly tree_cover_loss_prodes_wdpa_yearly tree_cover_loss_prodes_primary_forest_yearly country_area country_specific_deforestation_yearly country_specific_deforestation_wdpa_yearly country_specific_deforestation_landmark_yearly country_specific_deforestation_classified_region_yearly classified_region_area tree_cover_loss_brazil_biomes_yearly tree_cover_extent_total tree_cover_extent_primary_forest tree_cover_extent_protected_areas tree_cover_extent_peat tree_cover_extent_intact_forest natural_habitat_primary natural_habitat_intact_forest total_area protected_areas_area peat_area arg_otbn_area protected_areas_by_category_area landmark_by_category_area brazil_biomes idn_legal_area sea_landcover_area idn_landcover_area idn_forest_moratorium_area south_america_presence legal_amazon_presence brazil_biomes_presence cerrado_biome_presence southeast_asia_presence indonesia_presence argentina_presence commodity_value_forest_extent commodity_value_peat commodity_value_protected_areas commodity_threat_deforestation commodity_threat_peat commodity_threat_protected_areas commodity_threat_fires +12 2 2 {"2001":0.1529,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.1529,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0764,"2012":0.1529,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.5351,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.0} {} {} {} {"2001":0.1529,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.1529,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0764,"2012":0.1529,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.5351,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.0} {"COL":{"2001":0.1529,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.1529,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0764,"2012":0.1529,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.5351,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.0}} {"COL":{"Category II":{"2001":0.1529,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.1529,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0764,"2012":0.1529,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.5351,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.0}}} {"COL":{"Not Reported":{"2001":0.1529,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.1529,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0764,"2012":0.1529,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.5351,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.0}}} {"COL":{"Zona restricción;Restricciones legales":{"2001":0.1529,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.1529,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0764,"2012":0.1529,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.5351,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.0}}} {} {} {} {} {} {} {} {} {} {"COL":1792.6054} {} {} {} {} {"COL":{"Zona restricción;Restricciones legales":1792.6054}} {} 24.0046 0.0 24.0046 0.0 0.0 0.0 0.0 1792.6054 1792.6054 0.0 {} {"Category II":1792.6054} {"Not Reported":1792.6054} {} {} {} {} 0.0 true false false false false false false {"2001":0.4587,"2002":0.4587,"2003":0.4587,"2004":0.4587,"2005":0.4587,"2006":0.4587,"2007":0.4587,"2008":0.4587,"2009":0.3058,"2010":0.3058,"2011":0.3058,"2012":0.3058,"2013":0.3058,"2014":0.3058,"2015":0.3058,"2016":0.3058,"2017":0.3058,"2018":0.3058,"2019":0.3058,"2020":0.3058,"2021":0.3058,"2022":0.3058} {"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0} {"2001":1792.6054,"2002":1792.6054,"2003":1792.6054,"2004":1792.6054,"2005":1792.6054,"2006":1792.6054,"2007":1792.6054,"2008":1792.6054,"2009":1792.6054,"2010":1792.6054,"2011":1792.6054,"2012":1792.6054,"2013":1792.6054,"2014":1792.6054,"2015":1792.6054,"2016":1792.6054,"2017":1792.6054,"2018":1792.6054,"2019":1792.6054,"2020":1792.6054,"2021":1792.6054,"2022":1792.6054} {"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.1529,"2008":0.1529,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0} {"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0} {"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.1529,"2008":0.1529,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0} {} +12 1 2 {"2001":16.2101,"2002":4.9701,"2003":6.8816,"2004":8.0285,"2005":0.0,"2006":6.8816,"2007":1.9116,"2008":3.4408,"2009":7.7226,"2010":4.8936,"2011":0.4588,"2012":0.6117,"2013":0.0,"2014":0.6882,"2015":0.6117,"2016":1.2235,"2017":0.8411,"2018":2.0645,"2019":4.0526,"2020":0.6117,"2021":0.2294,"2022":2.7528,"2023":0.2294} {"2001":9.252,"2002":4.8936,"2003":4.4348,"2004":7.2638,"2005":0.0,"2006":5.8875,"2007":1.9116,"2008":2.3703,"2009":7.4932,"2010":4.2819,"2011":0.2294,"2012":0.4588,"2013":0.0,"2014":0.5352,"2015":0.0,"2016":0.0,"2017":0.0,"2018":1.1469,"2019":1.1469,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.2294} {} {} {"2001":15.4455,"2002":4.5877,"2003":1.0705,"2004":0.2294,"2005":0.0,"2006":0.4588,"2007":1.9116,"2008":0.0765,"2009":0.8411,"2010":0.4588,"2011":0.2294,"2012":0.3823,"2013":0.0,"2014":0.0,"2015":0.0,"2016":1.2235,"2017":0.6882,"2018":0.9176,"2019":2.9057,"2020":0.0,"2021":0.0,"2022":2.5998,"2023":0.2294} {"COL":{"2001":16.2101,"2002":4.9701,"2003":6.8816,"2004":8.0285,"2005":0.0,"2006":6.8816,"2007":1.9116,"2008":3.4408,"2009":7.7226,"2010":4.8936,"2011":0.4588,"2012":0.6117,"2013":0.0,"2014":0.6882,"2015":0.6117,"2016":1.2235,"2017":0.8411,"2018":2.0645,"2019":4.0526,"2020":0.6117,"2021":0.2294,"2022":2.7528,"2023":0.2294}} {"COL":{"Category VI":{"2001":0.5353,"2002":0.0,"2003":0.3823,"2004":0.1529,"2005":0.0,"2006":0.2294,"2007":0.0,"2008":0.0765,"2009":0.0765,"2010":0.1529,"2011":0.0,"2012":0.2294,"2013":0.0,"2014":0.0,"2015":0.0,"2016":1.2235,"2017":0.6882,"2018":0.9176,"2019":2.8292,"2020":0.0,"2021":0.0,"2022":2.5998,"2023":0.0},"Category II":{"2001":14.9102,"2002":4.5877,"2003":0.6882,"2004":0.0765,"2005":0.0,"2006":0.2294,"2007":1.9116,"2008":0.0,"2009":0.7646,"2010":0.3059,"2011":0.2294,"2012":0.1529,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0765,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.2294}}} {} {"COL":{"Zona restricción;Restricciones acuerdo cero deforestación":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0765,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.0},"Zona restricción;Restricciones técnicas (Áreas no agropecuarias)":{"2001":0.2294,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0765,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.0},"Zona restricción;Restricciones legales":{"2001":15.9043,"2002":4.9701,"2003":6.8816,"2004":8.0285,"2005":0.0,"2006":6.8051,"2007":1.9116,"2008":3.4408,"2009":7.7226,"2010":4.8936,"2011":0.3823,"2012":0.4588,"2013":0.0,"2014":0.6882,"2015":0.6117,"2016":1.2235,"2017":0.8411,"2018":2.0645,"2019":4.0526,"2020":0.6117,"2021":0.2294,"2022":2.7528,"2023":0.2294},"Condicionado;Ambiental":{"2001":0.0765,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.0},"Condicionado;Ambiental/Riesgo de desastres":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0765,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0765,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0,"2023":0.0}}} {} {} {} {} {} {} {} {} {} {"COL":1807.2221} {} {} {} {} {"COL":{"Zona restricción;Restricciones acuerdo cero deforestación":2.5998,"Zona restricción;Restricciones técnicas (Áreas no agropecuarias)":5.8876,"Zona restricción;Restricciones legales":1798.5053,"Condicionado;Ambiental":0.0765,"Condicionado;Ambiental/Riesgo de desastres":0.1529}} {} 1346.2938 607.9543 1020.7132 0.0 0.0 608.3367 0.0 1807.2221 1347.2942 0.0 {} {"Category VI":558.8093,"Category II":788.4849} {} {} {} {} {} 0.0 true false false false false false false {"2001":850.7308,"2002":850.7308,"2003":835.1324,"2004":830.2388,"2005":823.5866,"2006":815.864,"2007":815.864,"2008":810.1294,"2009":808.2943,"2010":805.0064,"2011":797.3603,"2012":792.5432,"2013":792.0844,"2014":791.4727,"2015":791.4727,"2016":790.7845,"2017":790.3257,"2018":789.5611,"2019":789.4846,"2020":787.6495,"2021":783.8263,"2022":783.3675} {"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0} {"2001":1347.2942,"2002":1347.2942,"2003":1347.2942,"2004":1347.2942,"2005":1347.2942,"2006":1347.2942,"2007":1347.2942,"2008":1347.2942,"2009":1347.2942,"2010":1347.2942,"2011":1347.2942,"2012":1347.2942,"2013":1347.2942,"2014":1347.2942,"2015":1347.2942,"2016":1347.2942,"2017":1347.2942,"2018":1347.2942,"2019":1347.2942,"2020":1347.2942,"2021":1347.2942,"2022":1347.2942} {"2002":20.492,"2003":11.5458,"2004":14.3748,"2005":7.7226,"2006":5.7346,"2007":7.5697,"2008":5.123,"2009":10.934,"2010":12.4633,"2011":5.2759,"2012":1.0705,"2013":0.6117,"2014":0.6882,"2015":1.1469,"2016":1.2234,"2017":0.8411,"2018":1.9116,"2019":5.6583,"2020":4.282,"2021":0.4588,"2022":2.1411} {"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.0,"2022":0.0} {"2002":19.3451,"2003":5.5818,"2004":1.1469,"2005":0.0765,"2006":0.3823,"2007":2.2174,"2008":1.9116,"2009":0.9175,"2010":1.2999,"2011":0.6882,"2012":0.6117,"2013":0.3823,"2014":0.0,"2015":0.0,"2016":0.7647,"2017":0.7647,"2018":0.9176,"2019":3.6703,"2020":2.7528,"2021":0.0,"2022":2.1411} {} diff --git a/src/test/resources/colombia.tsv b/src/test/resources/colombia.tsv new file mode 100755 index 00000000..63c5a47e --- /dev/null +++ b/src/test/resources/colombia.tsv @@ -0,0 +1,3 @@ +list_id location_id geom +12 1 01030000000100000005000000d0e968d00cfc51c07039360723b41940d0e968d00cfc51c0f08b0d5e338f194014cc475870f951c0f08b0d5e338f194014cc475870f951c07039360723b41940d0e968d00cfc51c07039360723b41940 +12 2 010300000001000000050000005552e0cadd0d52c020bc65695a271a405552e0cadd0d52c0001cde34ea051a40b88a9c63010b52c0001cde34ea051a40b88a9c63010b52c020bc65695a271a405552e0cadd0d52c020bc65695a271a40 diff --git a/src/test/resources/palm-32-fcd-output/part-00000-620b10f2-c761-4468-b2f2-ac1ce38059ae-c000.csv b/src/test/resources/palm-32-fcd-output/part-00000-620b10f2-c761-4468-b2f2-ac1ce38059ae-c000.csv deleted file mode 100644 index a335dfbd..00000000 --- a/src/test/resources/palm-32-fcd-output/part-00000-620b10f2-c761-4468-b2f2-ac1ce38059ae-c000.csv +++ /dev/null @@ -1,2 +0,0 @@ -list_id location_id status_code location_error tree_cover_loss_total_yearly tree_cover_loss_primary_forest_yearly tree_cover_loss_peat_yearly tree_cover_loss_intact_forest_yearly tree_cover_loss_protected_areas_yearly tree_cover_loss_by_country_yearly tree_cover_loss_by_country_wdpa_yearly tree_cover_loss_by_country_landmark_yearly tree_cover_loss_arg_otbn_yearly tree_cover_loss_sea_landcover_yearly tree_cover_loss_idn_landcover_yearly tree_cover_loss_soy_yearly tree_cover_loss_idn_legal_yearly tree_cover_loss_idn_forest_moratorium_yearly tree_cover_loss_prodes_yearly tree_cover_loss_prodes_wdpa_yearly tree_cover_loss_prodes_primary_forest_yearly country_specific_deforestation_yearly country_specific_deforestation_wdpa_yearly country_specific_deforestation_landmark_yearly country_specific_deforestation_classified_region_yearly tree_cover_loss_brazil_biomes_yearly tree_cover_extent_total tree_cover_extent_primary_forest tree_cover_extent_protected_areas tree_cover_extent_peat tree_cover_extent_intact_forest natural_habitat_primary natural_habitat_intact_forest total_area protected_areas_area peat_area arg_otbn_area protected_areas_by_category_area landmark_by_category_area brazil_biomes idn_legal_area sea_landcover_area idn_landcover_area idn_forest_moratorium_area south_america_presence legal_amazon_presence brazil_biomes_presence cerrado_biome_presence southeast_asia_presence indonesia_presence argentina_presence commodity_value_forest_extent commodity_value_peat commodity_value_protected_areas commodity_threat_deforestation commodity_threat_peat commodity_threat_protected_areas commodity_threat_fires -1 31 2 {"2001":1021.7622,"2002":851.014,"2003":310.1835,"2004":2169.8398,"2005":2325.3843,"2006":4162.4968,"2007":2968.7863,"2008":4015.4403,"2009":2002.9194,"2010":1173.7001,"2011":1703.6902,"2012":2838.0498,"2013":1841.7568,"2014":2468.7732,"2015":2028.9672,"2016":3344.8135,"2017":1026.7609,"2018":525.5327,"2019":618.7052,"2020":924.699,"2021":857.8225,"2022":560.0482,"2023":1033.0567} {"2001":154.8617,"2002":306.7253,"2003":92.3781,"2004":717.7405,"2005":1202.6952,"2006":1831.5766,"2007":1668.2764,"2008":1753.2317,"2009":797.282,"2010":454.5023,"2011":872.3613,"2012":1251.8543,"2013":1083.6799,"2014":1290.2177,"2015":1360.2574,"2016":2313.5001,"2017":286.2809,"2018":159.8557,"2019":162.3929,"2020":134.2652,"2021":167.4697,"2022":133.6506,"2023":166.9302} {"2001":557.4251,"2002":236.2539,"2003":71.8566,"2004":741.25,"2005":957.52,"2006":1229.3335,"2007":1037.5018,"2008":891.235,"2009":486.4665,"2010":363.5759,"2011":411.9212,"2012":1078.9246,"2013":862.5621,"2014":974.783,"2015":942.4571,"2016":1472.8429,"2017":211.3403,"2018":144.7173,"2019":148.7917,"2020":142.3323,"2021":122.7372,"2022":94.914,"2023":138.4895} {} {"2001":42.2692,"2002":228.1732,"2003":11.3743,"2004":3.9196,"2005":1.614,"2006":15.3711,"2007":4.4576,"2008":2.8437,"2009":4.765,"2010":7.9931,"2011":10.7597,"2012":8.1466,"2013":0.1537,"2014":8.5307,"2015":18.6758,"2016":139.2616,"2017":10.7596,"2018":0.3843,"2019":0.2306,"2020":0.0,"2021":0.0769,"2022":0.0,"2023":0.9991} {} {} {} {} {"Rubber plantation":{"2001":3.0745,"2002":16.5256,"2003":36.0493,"2004":66.1791,"2005":73.4812,"2006":25.9797,"2007":5.9184,"2008":56.571,"2009":47.7317,"2010":33.3581,"2011":21.9825,"2012":52.9583,"2013":11.9137,"2014":42.2742,"2015":34.2038,"2016":63.4883,"2017":10.6839,"2018":24.4423,"2019":22.1363,"2020":10.4533,"2021":25.826,"2022":26.1332,"2023":27.286},"Secondary forest":{"2001":240.1012,"2002":352.6874,"2003":51.186,"2004":522.8408,"2005":879.6014,"2006":1310.6826,"2007":981.6686,"2008":756.8744,"2009":359.2934,"2010":232.485,"2011":575.4717,"2012":1110.4372,"2013":787.2514,"2014":772.2979,"2015":966.528,"2016":1571.8466,"2017":149.9382,"2018":89.3794,"2019":136.8781,"2020":121.8915,"2021":68.6318,"2022":99.681,"2023":130.1926},"Agriculture":{"2001":3.151,"2002":9.1452,"2003":5.4563,"2004":53.8715,"2005":30.3561,"2006":22.9009,"2007":6.5323,"2008":10.9893,"2009":159.7649,"2010":38.7323,"2011":100.4403,"2012":104.3592,"2013":15.3698,"2014":35.8124,"2015":19.6734,"2016":38.1942,"2017":19.2886,"2018":10.5282,"2019":11.2197,"2020":7.9922,"2021":12.1419,"2022":9.2218,"2023":22.517},"Oil palm plantation":{"2001":389.5357,"2002":222.339,"2003":103.9797,"2004":96.4524,"2005":67.8614,"2006":368.7244,"2007":440.2632,"2008":428.9814,"2009":151.7946,"2010":184.5942,"2011":113.5139,"2012":263.0128,"2013":147.9443,"2014":88.3878,"2015":58.1061,"2016":70.7105,"2017":44.5029,"2018":31.2823,"2019":233.0475,"2020":526.052,"2021":395.9972,"2022":105.2237,"2023":484.6992},"Swamp":{"2001":265.2372,"2002":112.4372,"2003":38.2726,"2004":648.495,"2005":548.2747,"2006":855.4703,"2007":1129.3025,"2008":2086.08,"2009":484.4962,"2010":300.1085,"2011":526.378,"2012":478.8799,"2013":482.4034,"2014":742.3953,"2015":446.518,"2016":539.8938,"2017":620.8959,"2018":204.1215,"2019":105.5176,"2020":122.2736,"2021":197.9767,"2022":240.6287,"2023":233.0197},"Settlements":{"2001":0.1537,"2002":0.9992,"2003":0.0,"2004":0.6918,"2005":0.1537,"2006":1.1529,"2007":1.1529,"2008":0.538,"2009":1.0761,"2010":0.8455,"2011":1.1529,"2012":0.8454,"2013":0.0,"2014":0.6918,"2015":0.1537,"2016":0.2306,"2017":0.3843,"2018":0.0,"2019":0.1537,"2020":1.1529,"2021":1.3067,"2022":1.691,"2023":3.5357},"Grassland/shrub":{"2001":59.3337,"2002":89.231,"2003":37.5821,"2004":445.7701,"2005":432.4583,"2006":514.3995,"2007":235.9463,"2008":500.7963,"2009":334.6362,"2010":269.6786,"2011":186.2981,"2012":378.8895,"2013":330.4736,"2014":424.3189,"2015":165.2413,"2016":151.5619,"2017":77.7013,"2018":84.6964,"2019":29.59,"2020":91.3842,"2021":53.8004,"2022":41.7326,"2023":75.5499},"Primary forest":{"2001":41.1934,"2002":30.6653,"2003":13.68,"2004":98.6793,"2005":209.8123,"2006":379.429,"2007":115.8962,"2008":96.2208,"2009":368.2156,"2010":47.8819,"2011":42.0413,"2012":228.795,"2013":26.1305,"2014":255.8481,"2015":270.3755,"2016":823.8133,"2017":81.5399,"2018":47.9595,"2019":64.4845,"2020":22.7495,"2021":71.7856,"2022":5.7642,"2023":15.6791},"Water bodies":{"2001":0.8454,"2002":0.0768,"2003":0.0,"2004":0.1537,"2005":0.0,"2006":0.0,"2007":0.0769,"2008":0.0,"2009":0.0,"2010":0.1537,"2011":0.2306,"2012":0.6916,"2013":0.6917,"2014":0.6148,"2015":0.0,"2016":0.2306,"2017":0.0768,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.2305,"2022":0.1537,"2023":0.0769},"Mixed tree crops":{"2001":19.1363,"2002":16.9073,"2003":23.9776,"2004":236.7062,"2005":83.3852,"2006":683.7575,"2007":52.029,"2008":78.3891,"2009":95.9108,"2010":65.8624,"2011":136.1808,"2012":219.1809,"2013":39.5784,"2014":106.132,"2015":68.1674,"2016":84.8439,"2017":21.749,"2018":33.1229,"2019":15.6777,"2020":20.7498,"2021":30.1257,"2022":29.8185,"2023":40.5006}} {"Bare land":{"2001":3.8428,"2002":35.2766,"2003":5.3801,"2004":14.4491,"2005":17.6005,"2006":39.8116,"2007":99.1447,"2008":141.5687,"2009":59.9482,"2010":20.7508,"2011":136.3415,"2012":129.3478,"2013":94.2991,"2014":83.0794,"2015":280.0642,"2016":735.1371,"2017":28.9729,"2018":36.1198,"2019":8.3774,"2020":7.1477,"2021":8.0699,"2022":2.3824,"2023":16.8306},"Mining":{"2001":7.301,"2002":2.7666,"2003":5.2258,"2004":11.9889,"2005":15.2172,"2006":9.1456,"2007":7.6082,"2008":34.8914,"2009":16.9072,"2010":8.9918,"2011":12.4502,"2012":29.5112,"2013":1.0759,"2014":16.8304,"2015":2.5362,"2016":1.9982,"2017":1.7676,"2018":0.7685,"2019":0.3074,"2020":0.4611,"2021":1.7676,"2022":3.5353,"2023":4.3037},"Settlement":{"2001":30.2802,"2002":84.4598,"2003":6.7627,"2004":5.226,"2005":1.9982,"2006":15.5239,"2007":5.3029,"2008":146.7178,"2009":9.1456,"2010":5.9944,"2011":8.2999,"2012":20.9038,"2013":6.4557,"2014":10.4519,"2015":14.0641,"2016":20.5962,"2017":9.6834,"2018":5.9946,"2019":5.6871,"2020":7.5318,"2021":7.5314,"2022":7.7622,"2023":7.839},"Secondary forest":{"2001":14.8329,"2002":34.5077,"2003":10.3753,"2004":63.4026,"2005":86.5381,"2006":58.5628,"2007":81.0051,"2008":222.8806,"2009":92.1507,"2010":46.96,"2011":105.6723,"2012":258.1458,"2013":358.5935,"2014":604.2224,"2015":692.3818,"2016":1208.3837,"2017":259.4575,"2018":110.7479,"2019":151.2503,"2020":92.763,"2021":119.8204,"2022":109.8252,"2023":112.2107},"Agriculture":{"2001":87.6883,"2002":42.4224,"2003":18.9819,"2004":289.878,"2005":266.1325,"2006":746.3828,"2007":376.2672,"2008":177.9118,"2009":282.8805,"2010":120.1185,"2011":271.2771,"2012":638.4726,"2013":155.6217,"2014":248.7641,"2015":220.5643,"2016":382.8723,"2017":105.1299,"2018":55.5624,"2019":44.6495,"2020":56.5613,"2021":42.3446,"2022":44.4967,"2023":65.7837},"Swamp":{"2001":110.6023,"2002":161.6235,"2003":30.1276,"2004":349.3156,"2005":345.4651,"2006":346.3096,"2007":162.935,"2008":218.7309,"2009":146.9478,"2010":95.2199,"2011":132.4202,"2012":382.8885,"2013":159.6257,"2014":317.4138,"2015":296.5912,"2016":418.5593,"2017":115.2812,"2018":74.3201,"2019":85.1564,"2020":78.4694,"2021":128.1948,"2022":88.1532,"2023":81.8506},"Grassland/shrub":{"2001":4.9185,"2002":20.2891,"2003":11.7584,"2004":38.7334,"2005":22.748,"2006":135.7978,"2007":15.2937,"2008":74.7011,"2009":35.6594,"2010":20.4429,"2011":50.7993,"2012":105.518,"2013":11.4509,"2014":46.2648,"2015":55.7949,"2016":62.8662,"2017":264.4497,"2018":34.5065,"2019":9.8372,"2020":5.1492,"2021":6.9934,"2022":55.8711,"2023":47.8786},"Estate crop plantation":{"2001":759.8369,"2002":469.6682,"2003":221.0338,"2004":1396.0776,"2005":1569.454,"2006":2808.3496,"2007":2218.0015,"2008":2990.2754,"2009":1359.0495,"2010":854.7606,"2011":983.3553,"2012":1269.5731,"2013":1053.2509,"2014":1138.9796,"2015":466.1251,"2016":509.328,"2017":239.7129,"2018":205.5913,"2019":312.8252,"2020":676.3081,"2021":539.8723,"2022":247.7148,"2023":696.1293},"Body of water":{"2001":2.4593,"2002":0.0,"2003":0.538,"2004":0.7685,"2005":0.2306,"2006":2.6132,"2007":3.228,"2008":7.7625,"2009":0.2306,"2010":0.4611,"2011":3.0743,"2012":3.689,"2013":1.3834,"2014":2.7668,"2015":0.8454,"2016":5.0725,"2017":2.3057,"2018":1.9215,"2019":0.6148,"2020":0.3074,"2021":3.228,"2022":0.3074,"2023":0.2306}} {} {"Converted Production Forest":{"2001":151.8635,"2002":60.1778,"2003":69.0172,"2004":724.5834,"2005":1148.2139,"2006":1123.3127,"2007":1023.561,"2008":844.268,"2009":747.8878,"2010":275.2161,"2011":481.5731,"2012":804.1156,"2013":820.0067,"2014":1024.9196,"2015":1197.936,"2016":1866.668,"2017":356.367,"2018":233.3308,"2019":223.499,"2020":193.5994,"2021":241.5624,"2022":233.5622,"2023":188.2192},"Other Utilization Area":{"2001":712.0267,"2002":482.1867,"2003":221.5682,"2004":1414.5116,"2005":1126.5942,"2006":2837.7298,"2007":1853.8397,"2008":3013.7624,"2009":1165.5631,"2010":833.4598,"2011":1098.1437,"2012":1865.7614,"2013":971.0994,"2014":1259.078,"2015":657.4037,"2016":999.5492,"2017":622.2031,"2018":279.4429,"2019":376.9137,"2020":705.8893,"2021":595.4311,"2022":299.8162,"2023":789.0379},"Sanctuary Reserves/Nature Conservation Area":{"2001":42.2692,"2002":228.1732,"2003":11.3743,"2004":3.9196,"2005":1.614,"2006":15.3711,"2007":4.4576,"2008":2.8437,"2009":4.765,"2010":7.9931,"2011":10.7597,"2012":8.1466,"2013":0.1537,"2014":8.5307,"2015":18.6758,"2016":139.2616,"2017":10.7596,"2018":0.3843,"2019":0.2306,"2020":0.0,"2021":0.0769,"2022":0.0,"2023":0.9991},"Production Forest":{"2001":113.1434,"2002":80.4763,"2003":7.6858,"2004":26.0567,"2005":48.7316,"2006":183.5469,"2007":84.3149,"2008":147.2649,"2009":84.4729,"2010":56.57,"2011":110.1392,"2012":156.3372,"2013":49.1136,"2014":173.4782,"2015":154.1063,"2016":334.2621,"2017":35.1256,"2018":10.4532,"2019":17.4472,"2020":24.9028,"2021":17.5241,"2022":26.3625,"2023":54.5699}} {"2001":85.0014,"2002":248.2325,"2003":18.829,"2004":97.8293,"2005":96.2941,"2006":176.9875,"2007":138.7928,"2008":129.4126,"2009":109.4342,"2010":65.0144,"2011":100.5959,"2012":428.132,"2013":566.3779,"2014":467.2467,"2015":304.2577,"2016":712.6515,"2017":145.3232,"2018":56.2574,"2019":82.8502,"2020":54.0272,"2021":24.2097,"2022":14.7553,"2023":31.6631} {} {} {} {} {} {} {} {} 76338.8266 34513.678 6014.0986 23301.5138 0.0 34530.8164 0.0 125583.7284 6614.0107 31984.9079 {} {} {} {} {"Converted Production Forest":28600.5448,"Other Utilization Area":81822.5991,"Sanctuary Reserves/Nature Conservation Area":6614.0107,"Production Forest":4848.8827} {"Rubber plantation":3542.3364,"Secondary forest":24896.0268,"Agriculture":2763.3729,"Oil palm plantation":24398.5485,"Swamp":29043.6031,"Settlements":851.2415,"Grassland/shrub":22752.6953,"Primary forest":8261.1709,"Water bodies":3133.0348,"Mixed tree crops":5941.6982} {"Bare land":2902.1353,"Mining":1392.3433,"Settlement":5798.0268,"Secondary forest":21966.5842,"Agriculture":9963.5593,"Swamp":7625.7847,"Grassland/shrub":2875.049,"Estate crop plantation":69353.0242,"Body of water":3707.2217} 13342.6717 false false false false true true false {"2001":23485.8882,"2002":23485.8882,"2003":23374.1445,"2004":23119.9955,"2005":23068.1202,"2006":22694.5486,"2007":22285.1614,"2008":21463.0712,"2009":21006.0263,"2010":20669.6397,"2011":20292.7605,"2012":20136.9792,"2013":19754.7947,"2014":18881.5269,"2015":18606.9329,"2016":18125.0618,"2017":17132.4074,"2018":15378.6605,"2019":15120.9727,"2020":15003.7708,"2021":14849.0634,"2022":14732.6295} {"2001":31984.9079,"2002":31984.9079,"2003":31984.9079,"2004":31984.9079,"2005":31984.9079,"2006":31984.9079,"2007":31984.9079,"2008":31984.9079,"2009":31984.9079,"2010":31984.9079,"2011":31984.9079,"2012":31984.9079,"2013":31984.9079,"2014":31984.9079,"2015":31984.9079,"2016":31984.9079,"2017":31984.9079,"2018":31984.9079,"2019":31984.9079,"2020":31984.9079,"2021":31984.9079,"2022":31984.9079} {"2001":6614.0107,"2002":6614.0107,"2003":6614.0107,"2004":6614.0107,"2005":6614.0107,"2006":6614.0107,"2007":6614.0107,"2008":6614.0107,"2009":6614.0107,"2010":6614.0107,"2011":6614.0107,"2012":6614.0107,"2013":6614.0107,"2014":6614.0107,"2015":6614.0107,"2016":6614.0107,"2017":6614.0107,"2018":6614.0107,"2019":6614.0107,"2020":6614.0107,"2021":6614.0107,"2022":6614.0107} {"2002":365.8928,"2003":306.0243,"2004":425.4469,"2005":782.9588,"2006":1231.4774,"2007":1279.1351,"2008":793.4315,"2009":713.2658,"2010":532.6605,"2011":537.9658,"2012":1255.4523,"2013":1147.8618,"2014":756.465,"2015":1474.5255,"2016":2746.4013,"2017":2011.4348,"2018":374.8897,"2019":271.9092,"2020":271.1414,"2021":244.6289,"2022":225.7991} {"2002":14169.0317,"2003":14121.4598,"2004":14263.6318,"2005":14457.9871,"2006":14775.3894,"2007":14894.5153,"2008":14537.3838,"2009":14379.8313,"2010":14307.4367,"2011":14243.1121,"2012":14624.1389,"2013":14640.9716,"2014":14369.8445,"2015":14824.2131,"2016":15518.2929,"2017":15018.6597,"2018":14203.1553,"2019":14200.8532,"2020":14201.3905,"2021":14170.9561,"2022":14147.208} {"2002":223.0242,"2003":201.3516,"2004":10.5289,"2005":1.3833,"2006":4.6881,"2007":4.7649,"2008":1.0759,"2009":2.1519,"2010":3.7659,"2011":3.8427,"2012":4.1501,"2013":2.9204,"2014":1.3065,"2015":14.6794,"2016":75.0879,"2017":63.9436,"2018":3.1509,"2019":0.9222,"2020":0.9222,"2021":0.9222,"2022":0.9222} {} diff --git a/src/test/resources/palm-32-fcd-output/part-00000-f0e4ff7e-a350-48c1-8647-6ab1b9c184da-c000.csv b/src/test/resources/palm-32-fcd-output/part-00000-f0e4ff7e-a350-48c1-8647-6ab1b9c184da-c000.csv new file mode 100644 index 00000000..c2b49ffd --- /dev/null +++ b/src/test/resources/palm-32-fcd-output/part-00000-f0e4ff7e-a350-48c1-8647-6ab1b9c184da-c000.csv @@ -0,0 +1,2 @@ +list_id location_id status_code location_error tree_cover_loss_total_yearly tree_cover_loss_primary_forest_yearly tree_cover_loss_peat_yearly tree_cover_loss_intact_forest_yearly tree_cover_loss_protected_areas_yearly tree_cover_loss_by_country_yearly tree_cover_loss_by_country_wdpa_yearly tree_cover_loss_by_country_landmark_yearly tree_cover_loss_by_country_classified_region_yearly tree_cover_loss_arg_otbn_yearly tree_cover_loss_sea_landcover_yearly tree_cover_loss_idn_landcover_yearly tree_cover_loss_soy_yearly tree_cover_loss_idn_legal_yearly tree_cover_loss_idn_forest_moratorium_yearly tree_cover_loss_prodes_yearly tree_cover_loss_prodes_wdpa_yearly tree_cover_loss_prodes_primary_forest_yearly country_area country_specific_deforestation_yearly country_specific_deforestation_wdpa_yearly country_specific_deforestation_landmark_yearly country_specific_deforestation_classified_region_yearly classified_region_area tree_cover_loss_brazil_biomes_yearly tree_cover_extent_total tree_cover_extent_primary_forest tree_cover_extent_protected_areas tree_cover_extent_peat tree_cover_extent_intact_forest natural_habitat_primary natural_habitat_intact_forest total_area protected_areas_area peat_area arg_otbn_area protected_areas_by_category_area landmark_by_category_area brazil_biomes idn_legal_area sea_landcover_area idn_landcover_area idn_forest_moratorium_area south_america_presence legal_amazon_presence brazil_biomes_presence cerrado_biome_presence southeast_asia_presence indonesia_presence argentina_presence commodity_value_forest_extent commodity_value_peat commodity_value_protected_areas commodity_threat_deforestation commodity_threat_peat commodity_threat_protected_areas commodity_threat_fires +1 31 2 {"2001":1021.7622,"2002":851.014,"2003":310.1835,"2004":2169.8398,"2005":2325.3843,"2006":4162.4968,"2007":2968.7863,"2008":4015.4403,"2009":2002.9194,"2010":1173.7001,"2011":1703.6902,"2012":2838.0498,"2013":1841.7568,"2014":2468.7732,"2015":2028.9672,"2016":3344.8135,"2017":1026.7609,"2018":525.5327,"2019":618.7052,"2020":924.699,"2021":857.8225,"2022":560.0482,"2023":1033.0567} {"2001":154.8617,"2002":306.7253,"2003":92.3781,"2004":717.7405,"2005":1202.6952,"2006":1831.5766,"2007":1668.2764,"2008":1753.2317,"2009":797.282,"2010":454.5023,"2011":872.3613,"2012":1251.8543,"2013":1083.6799,"2014":1290.2177,"2015":1360.2574,"2016":2313.5001,"2017":286.2809,"2018":159.8557,"2019":162.3929,"2020":134.2652,"2021":167.4697,"2022":133.6506,"2023":166.9302} {"2001":557.4251,"2002":236.2539,"2003":71.8566,"2004":741.25,"2005":957.52,"2006":1229.3335,"2007":1037.5018,"2008":891.235,"2009":486.4665,"2010":363.5759,"2011":411.9212,"2012":1078.9246,"2013":862.5621,"2014":974.783,"2015":942.4571,"2016":1472.8429,"2017":211.3403,"2018":144.7173,"2019":148.7917,"2020":142.3323,"2021":122.7372,"2022":94.914,"2023":138.4895} {} {"2001":42.2692,"2002":228.1732,"2003":11.3743,"2004":3.9196,"2005":1.614,"2006":15.3711,"2007":4.4576,"2008":2.8437,"2009":4.765,"2010":7.9931,"2011":10.7597,"2012":8.1466,"2013":0.1537,"2014":8.5307,"2015":18.6758,"2016":139.2616,"2017":10.7596,"2018":0.3843,"2019":0.2306,"2020":0.0,"2021":0.0769,"2022":0.0,"2023":0.9991} {} {} {} {} {} {"Rubber plantation":{"2001":3.0745,"2002":16.5256,"2003":36.0493,"2004":66.1791,"2005":73.4812,"2006":25.9797,"2007":5.9184,"2008":56.571,"2009":47.7317,"2010":33.3581,"2011":21.9825,"2012":52.9583,"2013":11.9137,"2014":42.2742,"2015":34.2038,"2016":63.4883,"2017":10.6839,"2018":24.4423,"2019":22.1363,"2020":10.4533,"2021":25.826,"2022":26.1332,"2023":27.286},"Secondary forest":{"2001":240.1012,"2002":352.6874,"2003":51.186,"2004":522.8408,"2005":879.6014,"2006":1310.6826,"2007":981.6686,"2008":756.8744,"2009":359.2934,"2010":232.485,"2011":575.4717,"2012":1110.4372,"2013":787.2514,"2014":772.2979,"2015":966.528,"2016":1571.8466,"2017":149.9382,"2018":89.3794,"2019":136.8781,"2020":121.8915,"2021":68.6318,"2022":99.681,"2023":130.1926},"Agriculture":{"2001":3.151,"2002":9.1452,"2003":5.4563,"2004":53.8715,"2005":30.3561,"2006":22.9009,"2007":6.5323,"2008":10.9893,"2009":159.7649,"2010":38.7323,"2011":100.4403,"2012":104.3592,"2013":15.3698,"2014":35.8124,"2015":19.6734,"2016":38.1942,"2017":19.2886,"2018":10.5282,"2019":11.2197,"2020":7.9922,"2021":12.1419,"2022":9.2218,"2023":22.517},"Oil palm plantation":{"2001":389.5357,"2002":222.339,"2003":103.9797,"2004":96.4524,"2005":67.8614,"2006":368.7244,"2007":440.2632,"2008":428.9814,"2009":151.7946,"2010":184.5942,"2011":113.5139,"2012":263.0128,"2013":147.9443,"2014":88.3878,"2015":58.1061,"2016":70.7105,"2017":44.5029,"2018":31.2823,"2019":233.0475,"2020":526.052,"2021":395.9972,"2022":105.2237,"2023":484.6992},"Swamp":{"2001":265.2372,"2002":112.4372,"2003":38.2726,"2004":648.495,"2005":548.2747,"2006":855.4703,"2007":1129.3025,"2008":2086.08,"2009":484.4962,"2010":300.1085,"2011":526.378,"2012":478.8799,"2013":482.4034,"2014":742.3953,"2015":446.518,"2016":539.8938,"2017":620.8959,"2018":204.1215,"2019":105.5176,"2020":122.2736,"2021":197.9767,"2022":240.6287,"2023":233.0197},"Settlements":{"2001":0.1537,"2002":0.9992,"2003":0.0,"2004":0.6918,"2005":0.1537,"2006":1.1529,"2007":1.1529,"2008":0.538,"2009":1.0761,"2010":0.8455,"2011":1.1529,"2012":0.8454,"2013":0.0,"2014":0.6918,"2015":0.1537,"2016":0.2306,"2017":0.3843,"2018":0.0,"2019":0.1537,"2020":1.1529,"2021":1.3067,"2022":1.691,"2023":3.5357},"Grassland/shrub":{"2001":59.3337,"2002":89.231,"2003":37.5821,"2004":445.7701,"2005":432.4583,"2006":514.3995,"2007":235.9463,"2008":500.7963,"2009":334.6362,"2010":269.6786,"2011":186.2981,"2012":378.8895,"2013":330.4736,"2014":424.3189,"2015":165.2413,"2016":151.5619,"2017":77.7013,"2018":84.6964,"2019":29.59,"2020":91.3842,"2021":53.8004,"2022":41.7326,"2023":75.5499},"Primary forest":{"2001":41.1934,"2002":30.6653,"2003":13.68,"2004":98.6793,"2005":209.8123,"2006":379.429,"2007":115.8962,"2008":96.2208,"2009":368.2156,"2010":47.8819,"2011":42.0413,"2012":228.795,"2013":26.1305,"2014":255.8481,"2015":270.3755,"2016":823.8133,"2017":81.5399,"2018":47.9595,"2019":64.4845,"2020":22.7495,"2021":71.7856,"2022":5.7642,"2023":15.6791},"Water bodies":{"2001":0.8454,"2002":0.0768,"2003":0.0,"2004":0.1537,"2005":0.0,"2006":0.0,"2007":0.0769,"2008":0.0,"2009":0.0,"2010":0.1537,"2011":0.2306,"2012":0.6916,"2013":0.6917,"2014":0.6148,"2015":0.0,"2016":0.2306,"2017":0.0768,"2018":0.0,"2019":0.0,"2020":0.0,"2021":0.2305,"2022":0.1537,"2023":0.0769},"Mixed tree crops":{"2001":19.1363,"2002":16.9073,"2003":23.9776,"2004":236.7062,"2005":83.3852,"2006":683.7575,"2007":52.029,"2008":78.3891,"2009":95.9108,"2010":65.8624,"2011":136.1808,"2012":219.1809,"2013":39.5784,"2014":106.132,"2015":68.1674,"2016":84.8439,"2017":21.749,"2018":33.1229,"2019":15.6777,"2020":20.7498,"2021":30.1257,"2022":29.8185,"2023":40.5006}} {"Bare land":{"2001":3.8428,"2002":35.2766,"2003":5.3801,"2004":14.4491,"2005":17.6005,"2006":39.8116,"2007":99.1447,"2008":141.5687,"2009":59.9482,"2010":20.7508,"2011":136.3415,"2012":129.3478,"2013":94.2991,"2014":83.0794,"2015":280.0642,"2016":735.1371,"2017":28.9729,"2018":36.1198,"2019":8.3774,"2020":7.1477,"2021":8.0699,"2022":2.3824,"2023":16.8306},"Mining":{"2001":7.301,"2002":2.7666,"2003":5.2258,"2004":11.9889,"2005":15.2172,"2006":9.1456,"2007":7.6082,"2008":34.8914,"2009":16.9072,"2010":8.9918,"2011":12.4502,"2012":29.5112,"2013":1.0759,"2014":16.8304,"2015":2.5362,"2016":1.9982,"2017":1.7676,"2018":0.7685,"2019":0.3074,"2020":0.4611,"2021":1.7676,"2022":3.5353,"2023":4.3037},"Settlement":{"2001":30.2802,"2002":84.4598,"2003":6.7627,"2004":5.226,"2005":1.9982,"2006":15.5239,"2007":5.3029,"2008":146.7178,"2009":9.1456,"2010":5.9944,"2011":8.2999,"2012":20.9038,"2013":6.4557,"2014":10.4519,"2015":14.0641,"2016":20.5962,"2017":9.6834,"2018":5.9946,"2019":5.6871,"2020":7.5318,"2021":7.5314,"2022":7.7622,"2023":7.839},"Secondary forest":{"2001":14.8329,"2002":34.5077,"2003":10.3753,"2004":63.4026,"2005":86.5381,"2006":58.5628,"2007":81.0051,"2008":222.8806,"2009":92.1507,"2010":46.96,"2011":105.6723,"2012":258.1458,"2013":358.5935,"2014":604.2224,"2015":692.3818,"2016":1208.3837,"2017":259.4575,"2018":110.7479,"2019":151.2503,"2020":92.763,"2021":119.8204,"2022":109.8252,"2023":112.2107},"Agriculture":{"2001":87.6883,"2002":42.4224,"2003":18.9819,"2004":289.878,"2005":266.1325,"2006":746.3828,"2007":376.2672,"2008":177.9118,"2009":282.8805,"2010":120.1185,"2011":271.2771,"2012":638.4726,"2013":155.6217,"2014":248.7641,"2015":220.5643,"2016":382.8723,"2017":105.1299,"2018":55.5624,"2019":44.6495,"2020":56.5613,"2021":42.3446,"2022":44.4967,"2023":65.7837},"Swamp":{"2001":110.6023,"2002":161.6235,"2003":30.1276,"2004":349.3156,"2005":345.4651,"2006":346.3096,"2007":162.935,"2008":218.7309,"2009":146.9478,"2010":95.2199,"2011":132.4202,"2012":382.8885,"2013":159.6257,"2014":317.4138,"2015":296.5912,"2016":418.5593,"2017":115.2812,"2018":74.3201,"2019":85.1564,"2020":78.4694,"2021":128.1948,"2022":88.1532,"2023":81.8506},"Grassland/shrub":{"2001":4.9185,"2002":20.2891,"2003":11.7584,"2004":38.7334,"2005":22.748,"2006":135.7978,"2007":15.2937,"2008":74.7011,"2009":35.6594,"2010":20.4429,"2011":50.7993,"2012":105.518,"2013":11.4509,"2014":46.2648,"2015":55.7949,"2016":62.8662,"2017":264.4497,"2018":34.5065,"2019":9.8372,"2020":5.1492,"2021":6.9934,"2022":55.8711,"2023":47.8786},"Estate crop plantation":{"2001":759.8369,"2002":469.6682,"2003":221.0338,"2004":1396.0776,"2005":1569.454,"2006":2808.3496,"2007":2218.0015,"2008":2990.2754,"2009":1359.0495,"2010":854.7606,"2011":983.3553,"2012":1269.5731,"2013":1053.2509,"2014":1138.9796,"2015":466.1251,"2016":509.328,"2017":239.7129,"2018":205.5913,"2019":312.8252,"2020":676.3081,"2021":539.8723,"2022":247.7148,"2023":696.1293},"Body of water":{"2001":2.4593,"2002":0.0,"2003":0.538,"2004":0.7685,"2005":0.2306,"2006":2.6132,"2007":3.228,"2008":7.7625,"2009":0.2306,"2010":0.4611,"2011":3.0743,"2012":3.689,"2013":1.3834,"2014":2.7668,"2015":0.8454,"2016":5.0725,"2017":2.3057,"2018":1.9215,"2019":0.6148,"2020":0.3074,"2021":3.228,"2022":0.3074,"2023":0.2306}} {} {"Converted Production Forest":{"2001":151.8635,"2002":60.1778,"2003":69.0172,"2004":724.5834,"2005":1148.2139,"2006":1123.3127,"2007":1023.561,"2008":844.268,"2009":747.8878,"2010":275.2161,"2011":481.5731,"2012":804.1156,"2013":820.0067,"2014":1024.9196,"2015":1197.936,"2016":1866.668,"2017":356.367,"2018":233.3308,"2019":223.499,"2020":193.5994,"2021":241.5624,"2022":233.5622,"2023":188.2192},"Other Utilization Area":{"2001":712.0267,"2002":482.1867,"2003":221.5682,"2004":1414.5116,"2005":1126.5942,"2006":2837.7298,"2007":1853.8397,"2008":3013.7624,"2009":1165.5631,"2010":833.4598,"2011":1098.1437,"2012":1865.7614,"2013":971.0994,"2014":1259.078,"2015":657.4037,"2016":999.5492,"2017":622.2031,"2018":279.4429,"2019":376.9137,"2020":705.8893,"2021":595.4311,"2022":299.8162,"2023":789.0379},"Production Forest":{"2001":113.1434,"2002":80.4763,"2003":7.6858,"2004":26.0567,"2005":48.7316,"2006":183.5469,"2007":84.3149,"2008":147.2649,"2009":84.4729,"2010":56.57,"2011":110.1392,"2012":156.3372,"2013":49.1136,"2014":173.4782,"2015":154.1063,"2016":334.2621,"2017":35.1256,"2018":10.4532,"2019":17.4472,"2020":24.9028,"2021":17.5241,"2022":26.3625,"2023":54.5699},"Sanctuary Reserves/Nature Conservation Area":{"2001":42.2692,"2002":228.1732,"2003":11.3743,"2004":3.9196,"2005":1.614,"2006":15.3711,"2007":4.4576,"2008":2.8437,"2009":4.765,"2010":7.9931,"2011":10.7597,"2012":8.1466,"2013":0.1537,"2014":8.5307,"2015":18.6758,"2016":139.2616,"2017":10.7596,"2018":0.3843,"2019":0.2306,"2020":0.0,"2021":0.0769,"2022":0.0,"2023":0.9991}} {"2001":85.0014,"2002":248.2325,"2003":18.829,"2004":97.8293,"2005":96.2941,"2006":176.9875,"2007":138.7928,"2008":129.4126,"2009":109.4342,"2010":65.0144,"2011":100.5959,"2012":428.132,"2013":566.3779,"2014":467.2467,"2015":304.2577,"2016":712.6515,"2017":145.3232,"2018":56.2574,"2019":82.8502,"2020":54.0272,"2021":24.2097,"2022":14.7553,"2023":31.6631} {} {} {} {} {} {} {} {} {} {} 76338.8266 34513.678 6014.0986 23301.5138 0.0 34530.8164 0.0 125583.7284 6614.0107 31984.9079 {} {} {} {} {"Other Utilization Area":81822.5991,"Converted Production Forest":28600.5448,"Production Forest":4848.8827,"Sanctuary Reserves/Nature Conservation Area":6614.0107} {"Rubber plantation":3542.3364,"Secondary forest":24896.0268,"Agriculture":2763.3729,"Oil palm plantation":24398.5485,"Swamp":29043.6031,"Settlements":851.2415,"Grassland/shrub":22752.6953,"Primary forest":8261.1709,"Water bodies":3133.0348,"Mixed tree crops":5941.6982} {"Bare land":2902.1353,"Mining":1392.3433,"Settlement":5798.0268,"Secondary forest":21966.5842,"Agriculture":9963.5593,"Swamp":7625.7847,"Grassland/shrub":2875.049,"Estate crop plantation":69353.0242,"Body of water":3707.2217} 13342.6717 false false false false true true false {"2001":23485.8882,"2002":23485.8882,"2003":23374.1445,"2004":23119.9955,"2005":23068.1202,"2006":22694.5486,"2007":22285.1614,"2008":21463.0712,"2009":21006.0263,"2010":20669.6397,"2011":20292.7605,"2012":20136.9792,"2013":19754.7947,"2014":18881.5269,"2015":18606.9329,"2016":18125.0618,"2017":17132.4074,"2018":15378.6605,"2019":15120.9727,"2020":15003.7708,"2021":14849.0634,"2022":14732.6295} {"2001":31984.9079,"2002":31984.9079,"2003":31984.9079,"2004":31984.9079,"2005":31984.9079,"2006":31984.9079,"2007":31984.9079,"2008":31984.9079,"2009":31984.9079,"2010":31984.9079,"2011":31984.9079,"2012":31984.9079,"2013":31984.9079,"2014":31984.9079,"2015":31984.9079,"2016":31984.9079,"2017":31984.9079,"2018":31984.9079,"2019":31984.9079,"2020":31984.9079,"2021":31984.9079,"2022":31984.9079} {"2001":6614.0107,"2002":6614.0107,"2003":6614.0107,"2004":6614.0107,"2005":6614.0107,"2006":6614.0107,"2007":6614.0107,"2008":6614.0107,"2009":6614.0107,"2010":6614.0107,"2011":6614.0107,"2012":6614.0107,"2013":6614.0107,"2014":6614.0107,"2015":6614.0107,"2016":6614.0107,"2017":6614.0107,"2018":6614.0107,"2019":6614.0107,"2020":6614.0107,"2021":6614.0107,"2022":6614.0107} {"2002":365.8928,"2003":306.0243,"2004":425.4469,"2005":782.9588,"2006":1231.4774,"2007":1279.1351,"2008":793.4315,"2009":713.2658,"2010":532.6605,"2011":537.9658,"2012":1255.4523,"2013":1147.8618,"2014":756.465,"2015":1474.5255,"2016":2746.4013,"2017":2011.4348,"2018":374.8897,"2019":271.9092,"2020":271.1414,"2021":244.6289,"2022":225.7991} {"2002":14169.0317,"2003":14121.4598,"2004":14263.6318,"2005":14457.9871,"2006":14775.3894,"2007":14894.5153,"2008":14537.3838,"2009":14379.8313,"2010":14307.4367,"2011":14243.1121,"2012":14624.1389,"2013":14640.9716,"2014":14369.8445,"2015":14824.2131,"2016":15518.2929,"2017":15018.6597,"2018":14203.1553,"2019":14200.8532,"2020":14201.3905,"2021":14170.9561,"2022":14147.208} {"2002":223.0242,"2003":201.3516,"2004":10.5289,"2005":1.3833,"2006":4.6881,"2007":4.7649,"2008":1.0759,"2009":2.1519,"2010":3.7659,"2011":3.8427,"2012":4.1501,"2013":2.9204,"2014":1.3065,"2015":14.6794,"2016":75.0879,"2017":63.9436,"2018":3.1509,"2019":0.9222,"2020":0.9222,"2021":0.9222,"2022":0.9222} {} diff --git a/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala b/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala index bc3310ea..294cff40 100644 --- a/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala +++ b/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala @@ -20,8 +20,10 @@ class ForestChangeDiagnosticAnalysisSpec extends TestEnvironment with DataFrameC def antarcticaInputTsvPath = getClass.getResource("/antarctica.tsv").toString() def argentinaInputTsvPath = getClass.getResource("/argentina.tsv").toString() def argBraInputTsvPath = getClass.getResource("/argbra.tsv").toString() + def colInputTsvPath = getClass.getResource("/colombia.tsv").toString() def palm32ExpectedOutputPath = getClass.getResource("/palm-32-fcd-output").toString() def argBraExpectedOutputPath = getClass.getResource("/argbra-fcd-output").toString() + def colExpectedOutputPath = getClass.getResource("/col-fcd-output").toString() def FCD(features: RDD[ValidatedLocation[Geometry]]) = { val fireAlertsRdd = { @@ -163,4 +165,20 @@ class ForestChangeDiagnosticAnalysisSpec extends TestEnvironment with DataFrameC assertSmallDataFrameEquality(fcdDF, expectedDF) } + + it("matches recorded output for COL location", ProTag) { + val colRDD = ValidatedFeatureRDD( + NonEmptyList.one(colInputTsvPath), + "gfwpro", + FeatureFilter.empty, + splitFeatures = true + ) + val fcd = FCD(colRDD) + val fcdDF = ForestChangeDiagnosticDF.getFeatureDataFrame(fcd, spark) + //saveExpectedFcdResult(fcdDF, colExpectedOutputPath) + + val expectedDF = readExpectedFcdResult(colExpectedOutputPath) + + assertSmallDataFrameEquality(fcdDF, expectedDF) + } }