Skip to content

Commit

Permalink
GTC-2753 Add GADM test
Browse files Browse the repository at this point in the history
  • Loading branch information
@manukala6
manukala6 committed Apr 30, 2024
1 parent 9ee8fe9 commit 50f0296
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 19 deletions.
21 changes: 21 additions & 0 deletions .../resources/idn1_5Gadm-aum-output/part-00000-6550b168-100a-48ea-85c3-ab9dbad533d3-c000.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
iso adm1 adm2 umd_tree_cover_density_2000__threshold tsc_tree_cover_loss_drivers__driver is__birdlife_alliance_for_zero_extinction_sites gfw_planted_forests__type is__gmw_global_mangrove_extent_1996 is__gmw_global_mangrove_extent_2020 is__umd_regional_primary_forest_2001 is__wwf_tiger_conservation_landscapes is__landmark_indigenous_and_community_lands is__birdlife_key_biodiversity_areas is__gfw_mining_concessions is__gfw_peatlands is__gfw_oil_palm is__idn_forest_moratorium is__gfw_wood_fiber is__gfw_resource_rights is__gfw_managed_forests is__umd_tree_cover_gain is__ifl_intact_forest_landscapes_2000 wri_tropical_tree_cover__decile umd_global_land_cover__ipcc_class umd_tree_cover_density__threshold tsc_tree_cover_loss_drivers__type is__birdlife_alliance_for_zero_extinction_site gfw_plantation__type is__gmw_mangroves_1996 is__gmw_mangroves_2020 is__gfw_tiger_landscape is__landmark_land_right is__gfw_land_right is__birdlife_key_biodiversity_area is__gfw_mining is__peatland is__gfw_resource_right is__gfw_managed_forest is__umd_tree_cover_gain_2000-2012 wdpa_protected_areas__iucn_cat wdpa_protected_area__iucn_cat umd_tree_cover_extent_2000__ha umd_tree_cover_extent_2010__ha area__ha umd_tree_cover_gain__ha whrc_aboveground_biomass_stock_2000__Mg whrc_aboveground_co2_stock_2000__Mg umd_tree_cover_loss__ha whrc_aboveground_biomass_loss__Mg gfw_full_extent_aboveground_gross_removals__Mg_CO2 gfw_full_extent_belowground_gross_removals__Mg_CO2 gfw_full_extent_gross_removals__Mg_CO2 gfw_full_extent_net_flux__Mg_CO2e gfw_full_extent_gross_emissions_CO2_only__Mg_CO2 gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e gfw_full_extent_gross_emissions__Mg_CO2e gfw_soil_carbon_stocks_2000__Mg_C umd_tree_cover_loss_from_fires__ha wri_tropical_tree_cover_extent__ha umd_tree_cover_gain_2000-2012__ha gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg gfw_net_flux_co2e__Mg gfw_gross_emissions_co2e_all_gases__Mg gfw_aboveground_carbon_stocks_2000__Mg_C gfw_belowground_carbon_stocks_2000__Mg_C
IDN 1 5 10 Forestry false false false false false false false false true false true false false false true false -1 10 Forestry false false false false false false false false true false false true 17.130015189640698 21.047648719686006 24.65794337168607 24.65794337168607 1553.5853078468542 2677.3453471894118 4.148091720768501 337.9154835383149 2959.6821244664825 681.9686181397327 3641.6507426062158 -1331.7400461799593 2249.9517113175484 59.95898408331365 2309.9107000893478 731.9029698190467 0.3840750078701443 0.0 24.65794337168607 3641.6507426062158 -1331.7400461799593 2309.9107000893478 730.1850960064065 168.28818278130564
IDN 1 5 20 Forestry false false false false false false false false true false false false false false true false -1 20 Forestry false false false false false false false false true false false true 97.56069464542317 112.15397293188057 156.4775070288809 156.4775070288809 8943.02261492067 15411.808973046624 40.33144845001979 3590.176399981635 22481.96509690871 5014.218977743554 27496.184074652305 2335.7904834169335 27421.593691418355 2410.3808779312085 29831.974578727364 4731.5294211366845 17.0547989277691 0.0 156.4775070288809 27496.184074652305 2335.7904834169335 29831.974578727364 4203.220623081038 949.4447282628322
IDN 1 5 25 Forestry false false false false true false true false false false false false false false true false -1 25 Forestry false false false true false false true false false false false true 16.36470022039633 21.204984249800887 23.81717873118883 23.81717873118883 2348.297658547615 4046.8996315637205 2.9195226297635903 457.67347349983925 3806.0704583916913 788.6815980427451 4594.752056434439 -2858.88462042953 1725.0749110976662 10.792504684791684 1735.8674149032129 940.9319238407434 0.23048891370605468 0.0 23.81717873118883 4594.752056434439 -2858.88462042953 1735.8674149032129 1103.6998977002822 229.2391991615967
IDN 1 5 15 Forestry false false false false false false false false true false true false false false false false -1 15 Forestry false false false false false false false false true false false false 1497.2865354052078 1408.265593221533 1965.0484881352252 0.0 187809.2270003493 323657.90119727043 165.52913902380584 20026.16833387461 155616.03807895214 38484.00157614626 194100.03965510064 -123850.98203365019 67556.79536633485 2692.2620228375376 70249.05737217749 66146.28380857092 21.123766364439152 0.0 0.0 194100.03965510064 -123850.98203365019 70249.05737217749 88258.23198676288 22317.517101079728
IDN 1 5 50 Forestry false false false false false false true false true false false false false false true false -1 50 Forestry false false false false false false true false true false false true 108.88231100066876 124.71175384680967 179.72868151171528 179.72868151171528 11818.73054264031 20367.612301816793 54.17200751177828 6041.927615675271 35604.0447220966 8762.607014276608 44366.65173637323 -4858.80210178243 38341.176404641534 1166.6732597826556 39507.849674976555 6414.222136207407 6.91550885021064 0.0 179.72868151171528 44366.65173637323 -4858.80210178243 39507.849674976555 5554.803354419065 1353.8536488148784
IDN 1 5 20 Forestry false Fruit false false false false false false false false false false false false false true false -1 20 Forestry false Fruit false false false false false false false false false false true 6.9882228040161145 8.216830184037548 11.057879834628505 11.057879834628505 986.7454888318952 1700.4913924202988 0.3840591999270291 51.07923159100844 1877.626647542188 501.4263519385385 2379.052999480726 -2229.78496194034 149.26806360927586 0.0 149.26806360927586 394.0219582859453 0.0 0.0 11.057879834628505 2379.052999480726 -2229.78496194034 149.26806360927586 463.7703770204943 120.87259418193005
IDN 1 5 30 Forestry false false false false false false false false false false true false false false true false -1 30 Forestry false false false false false false false false false false false true 30.259406645652017 40.782024894388506 50.53707549449282 50.53707549449282 5543.558932052423 9553.399892903673 1.9201506563495552 339.86541092559264 7586.0424873078355 1925.3471082696003 9511.389595577442 -8130.9694521563615 1377.1372542716335 3.2829119971081835 1380.4201656828234 1952.8806413858633 0.07679747073676216 0.0 50.53707549449282 9511.389595577442 -8130.9694521563615 1380.4201656828234 2605.472698720353 671.6230123664216
IDN 1 5 -1 Forestry false Oil palm false false false false false false false false false false false false false true false 30 Forest -1 Forestry false Oil palm false false false false false false false false false false true 0.0 9.833161767273088 9.833161767273088 9.833161767273088 0.0 0.0 0.0 0.0 924.2878242031193 255.43410793707397 1179.7219321401924 -928.3440859478299 0.0 0.0 0.0 0.0 0.0 9.833161767273088 9.833161767273088 1179.7219321401924 -928.3440859478299 0.0 0.0 0.0
IDN 1 5 -1 Forestry false false false false false false true false false false true false false false true false 80 Forest -1 Forestry false false false false false false true false false false false true Category Ia/b or II Category Ia/b or II 0.0 0.537799265912679 0.537799265912679 0.537799265912679 0.0 0.0 0.0 0.0 91.02394755774478 19.254474495243585 110.27842205298836 -62.711787780534344 0.0 0.0 0.0 0.0 0.0 0.537799265912679 0.537799265912679 110.27842205298836 -62.711787780534344 0.0 0.0 0.0
IDN 1 5 10 Forestry false false false false false false true false false false true false false false false false -1 10 Forestry false false false false false false true false false false false false Category Ia/b or II Category Ia/b or II 130.99128373664817 133.21926071032274 135.9083379784392 0.0 27500.439491929897 47392.4240577592 11.754530042555745 2865.7180407352553 13025.13471559506 5150.87300660858 18176.007722203754 -11461.056035669018 6714.9512053873605 0.0 6714.9512053873605 6784.027897163175 0.0 0.0 0.0 18176.007722203754 -11461.056035669018 6714.9512053873605 12925.206544208666 5004.463883568725
IDN 1 5 50 Commodity driven deforestation false Unknown false false false false false true false true false true false false false false false -1 50 Commodity driven deforestation false Unknown false false false false false true false true false false false Other Category Other Category 7.530868559177247 7.684558516688909 7.684558516688909 0.0 1472.3634673279173 2537.3730420284446 6.070792407720948 1186.1882796768366 0.0 0.0 0.0 0.0 0.0 0.0 0.0 413.5830180777769 0.6916072024495442 0.0 0.0 0.0 0.0 0.0 692.0108276507656 234.92291608637368
IDN 1 5 0 Forestry false Oil palm false false false false false false false false false false false false false true false -1 0 Forestry false Oil palm false false false false false false false false false false true 236.61219005046917 236.61219005046917 236.61219005046917 236.61219005046917 12419.289570169916 21402.575692592753 17.745271145792785 1627.961346268703 21767.095873653707 5838.948385063617 27606.044258717262 -21886.42028629855 5700.594886645625 19.029127383426296 5719.624012856728 13750.980881579837 1.0755578231892904 0.0 236.61219005046917 27606.044258717262 -21886.42028629855 5719.624012856728 5837.066089910393 1650.496116716602
IDN 1 5 -1 Forestry false Oil palm false false false true false false false false false false false false false false false 70 Forest -1 Forestry false Oil palm false false true false false false false false false false false 0.0 0.9986839327990724 0.9986839327990724 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.9986839327990724 0.0 0.0 0.0 0.0 0.0 0.0
IDN 1 5 -1 Forestry false false false false true false true false true false false false false false false false 10 Grassland -1 Forestry false false false true false false true false true false false false 0.0 0.5378101345316569 0.5378101345316569 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5378101345316569 0.0 0.0 0.0 0.0 0.0 0.0
IDN 1 5 -1 Forestry false false false false false false true false true false true false false false false false 90 Settlements -1 Forestry false false false false false false true false true false false false 0.0 0.7684167158938261 0.7684167158938261 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.7684167158938261 0.0 0.0 0.0 0.0 0.0 0.0
IDN 1 5 -1 Forestry false false false false true false true false false true false false false false true false 90 Forest -1 Forestry false false false true false false true false false false false true 0.0 0.8451396069095866 0.8451396069095866 0.8451396069095866 0.0 0.0 0.0 0.0 123.88520624822482 27.202569161717292 151.0877754099421 -151.08777453068222 0.0 0.0 0.0 0.0 0.0 0.8451396069095866 0.8451396069095866 151.0877754099421 -151.08777453068222 0.0 0.0 0.0
IDN 1 5 -1 Unknown false Fruit false false false true false true false false false true false false false false true 50 Forest -1 Unknown false Fruit false false true false false true false false false false false Category Ia/b or II Category Ia/b or II 0.0 0.07677312813056099 0.07677312813056099 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.07677312813056099 0.0 0.0 0.0 0.0 0.0 0.0
IDN 1 5 15 Forestry false Fruit false false true false false true false true false true false false false false false -1 15 Forestry false Fruit false false false false false true false true false false false 0.15367689269748264 0.15367689269748264 0.15367689269748264 0.0 39.49496126023763 68.06298323847619 0.15367689269748264 39.49496126023763 7.790957213351335 4.661993873185977 12.452951086537311 182.94344870053723 172.5660188361291 22.830379485368873 195.3963983214975 8.913259776453994 0.15367689269748264 0.0 0.0 12.452951086537311 182.94344870053723 195.3963983214975 18.56263134677625 11.107607874642314
IDN 1 5 -1 Forestry false false false false false false true false false false true false false false true false 10 Forest -1 Forestry false false false false false false true false false false false true Other Category Other Category 0.0 0.6914794842983669 0.6914794842983669 0.6914794842983669 0.0 0.0 0.0 0.0 188.08974364313397 47.83736069389634 235.9271043370303 60.86110454028223 0.0 0.0 0.0 0.0 0.0 0.6914794842983669 0.6914794842983669 235.9271043370303 60.86110454028223 0.0 0.0 0.0
IDN 1 5 30 Forestry false Fruit false false false true false true false false false true false false false false true -1 30 Forestry false Fruit false false true false false true false false false false false 0.15359689346218533 0.230395340193278 0.230395340193278 0.0 41.547959681521135 71.60098385115475 0.0 0.0 23.439029845668998 5.187847808780273 28.62687765444927 -28.62687882630057 0.0 0.0 0.0 13.362929731210123 0.0 0.0 0.0 28.62687765444927 -28.62687882630057 0.0 19.5275413784333 4.322103404627438
95 changes: 76 additions & 19 deletions ...ala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSpec.scala
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package org.globalforestwatch.summarystats.annualupdate_minimal

import com.github.mrpowers.spark.fast.tests.DataFrameComparer
import geotrellis.vector._
import org.apache.spark.rdd.RDD
import cats.data.NonEmptyList
import org.apache.spark.sql.{DataFrame, SaveMode}
Expand All @@ -12,15 +11,38 @@ import org.globalforestwatch.features.{FeatureId, FeatureRDD, FeatureFilter}
class AnnualUpdateMinimalSpec extends TestEnvironment with DataFrameComparer {
def idn1_5GadmInputTsvPath = getClass.getResource("/idn1_5Gadm.tsv").toString()
def idn1_5GadmExpectedOutputPath = getClass.getResource("/idn1_5Gadm-aum-output").toString()
def wdpaInputTsvPath = getClass.getResource("/mulanje.tsv").toString()


val csvOptions: Map[String, String] = Map(
"header" -> "true",
"delimiter" -> "\t",
"escape" -> "\"",
"quoteMode" -> "MINIMAL",
"nullValue" -> "",
"emptyValue" -> ""
)

def AnnualUpdateMinimal(features: RDD[Feature[Geometry, FeatureId]]): Unit = {
AnnualUpdateMinimalAnalysis(
features,
"gadm",
spark,
kwargs = Map("config" -> GfwConfig.get())
)
def AnnualUpdateMinimal(inputPath: String, featureType: String): DataFrame = {

// Create featureRDD
val featureRDD = FeatureRDD(
NonEmptyList.one(inputPath),
featureType,
FeatureFilter.empty,
splitFeatures = true,
spark)

// Run AnnualUpdateMinimal Analysis
val kwargs = Map("config" -> GfwConfig.get())
val aumRDD: RDD[(FeatureId, AnnualUpdateMinimalSummary)] =
AnnualUpdateMinimalRDD(featureRDD, AnnualUpdateMinimalGrid.blockTileGrid, kwargs)
val aumDFFactory = AnnualUpdateMinimalDFFactory(featureType, aumRDD, spark)
val aumDF: DataFrame = aumDFFactory.getDataFrame

aumDF
}

/** Function to update expected results when this test becomes invalid */
def saveExpectedAumResult(aum: DataFrame, outputPath: String): Unit = {
aum.repartition(1)
Expand All @@ -29,21 +51,56 @@ class AnnualUpdateMinimalSpec extends TestEnvironment with DataFrameComparer {
.options(AnnualUpdateMinimalExport.csvOptions)
.csv(path = idn1_5GadmExpectedOutputPath)
}

def readAumResult(path: String) = {
spark.read
.options(Map(
"header" -> "true",
"delimiter" -> "\t",
"escape" -> "\"",
"quoteMode" -> "MINIMAL",
"nullValue" -> null,
"emptyValue" -> null,
"inferSchema" -> "true"
))
.csv(path)
}

it("matches recorded output for GADM") {
val gadmDF: DataFrame = AnnualUpdateMinimal(idn1_5GadmInputTsvPath, "gadm")

// Export and save results
import spark.implicits._
val exportDF = gadmDF
.transform(
AnnualUpdateMinimalDF.unpackValues(
List($"id.iso" as "iso", $"id.adm1" as "adm1", $"id.adm2" as "adm2")
)
)
val adm2DF: DataFrame = exportDF.transform(
AnnualUpdateMinimalDF.aggSummary(List("iso", "adm1", "adm2"))
)
val top20Rows = adm2DF.limit(20)
top20Rows
.write
.options(csvOptions)
.csv("output/idntest")

// Read expected results and compare
val expectedDF = readAumResult(idn1_5GadmExpectedOutputPath)
val top20RowsDF = readAumResult("output/idntest")
top20RowsDF.show()
expectedDF.show()

assertSmallDataFrameEquality(top20RowsDF, expectedDF)

def readExpectedFcdResult = {
val csvFile = spark.read
.options(AnnualUpdateMinimalExport.csvOptions)
.csv(idn1_5GadmExpectedOutputPath)
}

it("matches recorded output for first partition") {
val idn1_5GadmRDD = FeatureRDD(
NonEmptyList.one(idn1_5GadmInputTsvPath),
"gadm",
filters = FeatureFilter.empty,
splitFeatures = true,
spark)
it("matches recorded output for WDPA") {
val wdpaDF: DataFrame = AnnualUpdateMinimal(idn1_5GadmInputTsvPath, "wdpa")

}

}

// TODO: export dataframe, clean columns, compare with real results

0 comments on commit 50f0296

Please sign in to comment.