Merge pull request #147 from wri/develop

Merge develop to master

src/main/scala/org/globalforestwatch/config/GfwConfig.scala

@@ -14,7 +14,7 @@ object GfwConfig extends LazyLogging {
 
   def isGfwPro: Boolean = featureFlag == Some("pro")
 
-  lazy val get: GfwConfig = read(featureFlag.getOrElse("flagship"))
+  lazy val get: GfwConfig = read(featureFlag.getOrElse("default"))
 
   def read(flag: String): GfwConfig = {
     val rasterCatalogFile = s"raster-catalog-$flag.json"

src/main/scala/org/globalforestwatch/layers/TigerLandscapes.scala

@@ -5,7 +5,7 @@ import org.globalforestwatch.grids.GridTile
 case class TigerLandscapes(gridTile: GridTile, kwargs: Map[String, Any])
   extends BooleanLayer
     with OptionalILayer {
-  val datasetName = "gfw_tiger_landscapes"
+  val datasetName = "wwf_tiger_conservation_landscapes"
   val uri: String =
     uriForGrid(gridTile)
 }

src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala

@@ -7,20 +7,20 @@ import org.apache.spark.sql.{Column, DataFrame, SparkSession}
 object AnnualUpdateMinimalDF {
 
   val contextualLayers = List(
-    "umd_tree_cover_density__threshold",
-    "tsc_tree_cover_loss_drivers__type",
+    "umd_tree_cover_density_2000__threshold",
+    "tsc_tree_cover_loss_drivers__driver",
     "esa_land_cover_2015__class",
     "is__birdlife_alliance_for_zero_extinction_sites",
-    "gfw_plantations__type",
+    "gfw_planted_forests__type",
     "is__gmw_global_mangrove_extent_1996",
     "is__gmw_global_mangrove_extent_2016",
     "ifl_intact_forest_landscapes__year",
     "is__umd_regional_primary_forest_2001",
-    "is__gfw_tiger_landscapes",
+    "is__wwf_tiger_conservation_landscapes",
     "is__landmark_indigenous_and_community_lands",
     "is__gfw_land_rights",
     "is__birdlife_key_biodiversity_areas",
-    "is__gfw_mining",
+    "is__gfw_mining_concessions",
     "is__gfw_peatlands",
     "is__gfw_oil_palm",
     "is__idn_forest_moratorium",
@@ -39,20 +39,20 @@ object AnnualUpdateMinimalDF {
     def defaultUnpackCols =
       List(
         $"data_group.lossYear" as "umd_tree_cover_loss__year",
-        $"data_group.threshold" as "umd_tree_cover_density__threshold",
-        $"data_group.drivers" as "tsc_tree_cover_loss_drivers__type",
+        $"data_group.threshold" as "umd_tree_cover_density_2000__threshold",
+        $"data_group.drivers" as "tsc_tree_cover_loss_drivers__driver",
         $"data_group.globalLandCover" as "esa_land_cover_2015__class",
         $"data_group.primaryForest" as "is__umd_regional_primary_forest_2001",
         $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_sites",
-        $"data_group.plantations" as "gfw_plantations__type",
+        $"data_group.plantations" as "gfw_planted_forests__type",
         $"data_group.mangroves1996" as "is__gmw_global_mangrove_extent_1996",
         $"data_group.mangroves2016" as "is__gmw_global_mangrove_extent_2016",
         $"data_group.intactForestLandscapes" as "ifl_intact_forest_landscapes__year",
-        $"data_group.tigerLandscapes" as "is__gfw_tiger_landscapes",
+        $"data_group.tigerLandscapes" as "is__wwf_tiger_conservation_landscapes",
         $"data_group.landmark" as "is__landmark_indigenous_and_community_lands",
         $"data_group.landRights" as "is__gfw_land_rights",
         $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas",
-        $"data_group.mining" as "is__gfw_mining",
+        $"data_group.mining" as "is__gfw_mining_concessions",
         $"data_group.peatlands" as "is__gfw_peatlands",
         $"data_group.oilPalm" as "is__gfw_oil_palm",
         $"data_group.idnForestMoratorium" as "is__idn_forest_moratorium",
@@ -179,22 +179,22 @@ object AnnualUpdateMinimalDF {
     import spark.implicits._
 
     val defaultAggCols = List(
-      max(length($"tsc_tree_cover_loss_drivers__type")).cast("boolean") as "tsc_tree_cover_loss_drivers__type",
+      max(length($"tsc_tree_cover_loss_drivers__driver")).cast("boolean") as "tsc_tree_cover_loss_drivers__driver",
       max(length($"esa_land_cover_2015__class"))
         .cast("boolean") as "esa_land_cover_2015__class",
       max($"is__umd_regional_primary_forest_2001") as "is__umd_regional_primary_forest_2001",
       max($"is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites",
-      max(length($"gfw_plantations__type"))
-        .cast("boolean") as "gfw_plantations__type",
+      max(length($"gfw_planted_forests__type"))
+        .cast("boolean") as "gfw_planted_forests__type",
       max($"is__gmw_global_mangrove_extent_1996") as "is__gmw_global_mangrove_extent_1996",
       max($"is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016",
       max(length($"ifl_intact_forest_landscapes__year"))
         .cast("boolean") as "ifl_intact_forest_landscapes__year",
-      max($"is__gfw_tiger_landscapes") as "is__gfw_tiger_landscapes",
+      max($"is__wwf_tiger_conservation_landscapes") as "is__wwf_tiger_conservation_landscapes",
       max($"is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands",
       max($"is__gfw_land_rights") as "is__gfw_land_rights",
       max($"is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas",
-      max($"is__gfw_mining") as "is__gfw_mining",
+      max($"is__gfw_mining_concessions") as "is__gfw_mining_concessions",
       max($"is__gfw_peatlands") as "is__gfw_peatlands",
       max($"is__gfw_oil_palm") as "is__gfw_oil_palm",
       max($"is__idn_forest_moratorium") as "is__idn_forest_moratorium",
@@ -224,19 +224,19 @@ object AnnualUpdateMinimalDF {
     import spark.implicits._
 
     val defaultAggCols: List[Column] = List(
-      max($"tsc_tree_cover_loss_drivers__type") as "tsc_tree_cover_loss_drivers__type",
+      max($"tsc_tree_cover_loss_drivers__driver") as "tsc_tree_cover_loss_drivers__driver",
       max($"esa_land_cover_2015__class") as "esa_land_cover_2015__class",
       max($"is__umd_regional_primary_forest_2001") as "is__umd_regional_primary_forest_2001",
       max($"is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites",
-      max($"gfw_plantations__type") as "gfw_plantations__type",
+      max($"gfw_planted_forests__type") as "gfw_planted_forests__type",
       max($"is__gmw_global_mangrove_extent_1996") as "is__gmw_global_mangrove_extent_1996",
       max($"is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016",
       max($"ifl_intact_forest_landscapes__year") as "ifl_intact_forest_landscapes__year",
-      max($"is__gfw_tiger_landscapes") as "is__gfw_tiger_landscapes",
+      max($"is__wwf_tiger_conservation_landscapes") as "is__wwf_tiger_conservation_landscapes",
       max($"is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands",
       max($"is__gfw_land_rights") as "is__gfw_land_rights",
       max($"is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas",
-      max($"is__gfw_mining") as "is__gfw_mining",
+      max($"is__gfw_mining_concessions") as "is__gfw_mining_concessions",
       max($"is__gfw_peatlands") as "is__gfw_peatlands",
       max($"is__gfw_oil_palm") as "is__gfw_oil_palm",
       max($"is__idn_forest_moratorium") as "is__idn_forest_moratorium",

src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDownloadDF.scala

@@ -16,7 +16,7 @@ object AnnualUpdateMinimalDownloadDF {
     val yearRange = treecoverLossMinYear to treecoverLossMaxYear
 
     val annualDF = df
-      .groupBy($"iso", $"adm1", $"adm2", $"umd_tree_cover_density__threshold")
+      .groupBy($"iso", $"adm1", $"adm2", $"umd_tree_cover_density_2000__threshold")
       .pivot("umd_tree_cover_loss__year", yearRange)
       .agg(
         sum("umd_tree_cover_loss__ha") as "umd_tree_cover_loss__ha",
@@ -26,7 +26,7 @@ object AnnualUpdateMinimalDownloadDF {
       .na.fill(0, Seq("adm1", "adm2"))
 
     val totalDF = df
-      .groupBy($"iso", $"adm1", $"adm2", $"umd_tree_cover_density__threshold")
+      .groupBy($"iso", $"adm1", $"adm2", $"umd_tree_cover_density_2000__threshold")
       .agg(
         sum("umd_tree_cover_extent_2000__ha") as "umd_tree_cover_extent_2000__ha",
         sum("umd_tree_cover_extent_2010__ha") as "umd_tree_cover_extent_2010__ha",
@@ -46,7 +46,7 @@ object AnnualUpdateMinimalDownloadDF {
     totalDF
       .join(
         annualDF,
-        Seq("iso", "adm1", "adm2", "umd_tree_cover_density__threshold"),
+        Seq("iso", "adm1", "adm2", "umd_tree_cover_density_2000__threshold"),
         "inner"
       )
       .transform(setNullZero)
@@ -123,7 +123,7 @@ object AnnualUpdateMinimalDownloadDF {
 
     df.groupBy(
       groupByCols.head,
-      groupByCols.tail ::: List("umd_tree_cover_density__threshold"): _*
+      groupByCols.tail ::: List("umd_tree_cover_density_2000__threshold"): _*
     )
       .agg(aggCols.head, aggCols.tail: _*)
       .na.fill(0, Seq("avg_whrc_aboveground_biomass_2000_Mg_ha-1"))
@@ -185,7 +185,7 @@ object AnnualUpdateMinimalDownloadDF {
     import spark.implicits._
 
     val cols = List(
-      $"umd_tree_cover_density__threshold",
+      $"umd_tree_cover_density_2000__threshold",
       round($"umd_tree_cover_extent_2000__ha") as "umd_tree_cover_extent_2000__ha",
       round($"umd_tree_cover_extent_2010__ha") as "umd_tree_cover_extent_2010__ha",
       round($"area__ha") as "area__ha",
@@ -229,7 +229,7 @@ object AnnualUpdateMinimalDownloadDF {
    */
   private def removeCarbonThresholds(df: DataFrame): DataFrame = {
     def setNull(column: Column): Column =
-      when(df("umd_tree_cover_density__threshold") < 30, null).otherwise(column)
+      when(df("umd_tree_cover_density_2000__threshold") < 30, null).otherwise(column)
 
     val totalGrossEmissionsCo2eAllGasesCols =
       (for (i <- treecoverLossMinYear to treecoverLossMaxYear) yield {

src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala

@@ -146,7 +146,7 @@ object AnnualUpdateMinimalExport extends SummaryExport {
         $"country",
         $"subnational1",
         $"subnational2",
-        $"umd_tree_cover_density__threshold"
+        $"umd_tree_cover_density_2000__threshold"
       )
       .write
       .options(csvOptions)
@@ -163,7 +163,7 @@ object AnnualUpdateMinimalExport extends SummaryExport {
           .roundDownload2(List($"iso" as "country", $"adm1" as "subnational1"))
       )
       .coalesce(1)
-      .orderBy($"country", $"subnational1", $"umd_tree_cover_density__threshold")
+      .orderBy($"country", $"subnational1", $"umd_tree_cover_density_2000__threshold")
       .write
       .options(csvOptions)
       .csv(path = outputUrl + "/adm1/download")
@@ -179,7 +179,7 @@ object AnnualUpdateMinimalExport extends SummaryExport {
           .roundDownload2(List($"iso" as "country"))
       )
       .coalesce(1)
-      .orderBy($"country", $"umd_tree_cover_density__threshold")
+      .orderBy($"country", $"umd_tree_cover_density_2000__threshold")
       .write
       .options(csvOptions)
       .csv(path = outputUrl + "/iso/download")

src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala

@@ -7,12 +7,12 @@ import org.apache.spark.sql.{Column, DataFrame}
 object FireAlertsDF {
 
   val contextualLayers: List[String] = List(
-    "umd_tree_cover_density__threshold",
+    "umd_tree_cover_density_2000__threshold",
     "is__umd_regional_primary_forest_2001",
     "is__birdlife_alliance_for_zero_extinction_sites",
     "is__birdlife_key_biodiversity_areas",
     "is__landmark_indigenous_and_community_lands",
-    "gfw_plantations__type",
+    "gfw_planted_forests__type",
     "is__gfw_mining_concessions",
     "is__gfw_managed_forests",
     "rspo_oil_palm__certification_status",
@@ -38,12 +38,12 @@ object FireAlertsDF {
 
     def defaultCols =
       List(
-        $"data_group.threshold" as "umd_tree_cover_density__threshold",
+        $"data_group.threshold" as "umd_tree_cover_density_2000__threshold",
         $"data_group.primaryForest" as "is__umd_regional_primary_forest_2001",
         $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_sites",
         $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas",
         $"data_group.landmark" as "is__landmark_indigenous_and_community_lands",
-        $"data_group.plantations" as "gfw_plantations__type",
+        $"data_group.plantations" as "gfw_planted_forests__type",
         $"data_group.mining" as "is__gfw_mining_concessions",
         $"data_group.logging" as "is__gfw_managed_forests",
         $"data_group.rspo" as "rspo_oil_palm__certification_status",
@@ -158,8 +158,8 @@ object FireAlertsDF {
       max("is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites",
       max("is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas",
       max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands",
-      max(length($"gfw_plantations__type"))
-        .cast("boolean") as "gfw_plantations__type",
+      max(length($"gfw_planted_forests__type"))
+        .cast("boolean") as "gfw_planted_forests__type",
       max("is__gfw_mining_concessions") as "is__gfw_mining_concessions",
       max("is__gfw_managed_forests") as "is__gfw_managed_forests",
       max(length($"rspo_oil_palm__certification_status"))
@@ -196,7 +196,7 @@ object FireAlertsDF {
       max("is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites",
       max("is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas",
       max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands",
-      max("gfw_plantations__type") as "gfw_plantations__type",
+      max("gfw_planted_forests__type") as "gfw_planted_forests__type",
       max("is__gfw_mining_concessions") as "is__gfw_mining_concessions",
       max("is__gfw_managed_forests") as "is__gfw_managed_forests",
       max("rspo_oil_palm__certification_status") as "rspo_oil_palm__certification_status",

src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala

@@ -11,7 +11,7 @@ object GladAlertsDF {
     "is__birdlife_alliance_for_zero_extinction_sites",
     "is__birdlife_key_biodiversity_areas",
     "is__landmark_indigenous_and_community_lands",
-    "gfw_plantations__type",
+    "gfw_planted_forests__type",
     "is__gfw_mining_concessions",
     "is__gfw_managed_forests",
     "rspo_oil_palm__certification_status",
@@ -44,7 +44,7 @@ object GladAlertsDF {
         $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_sites",
         $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas",
         $"data_group.landmark" as "is__landmark_indigenous_and_community_lands",
-        $"data_group.plantations" as "gfw_plantations__type",
+        $"data_group.plantations" as "gfw_planted_forests__type",
         $"data_group.mining" as "is__gfw_mining_concessions",
         $"data_group.logging" as "is__gfw_managed_forests",
         $"data_group.rspo" as "rspo_oil_palm__certification_status",
@@ -126,8 +126,8 @@ object GladAlertsDF {
       max("is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites",
       max("is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas",
       max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands",
-      max(length($"gfw_plantations__type"))
-        .cast("boolean") as "gfw_plantations__type",
+      max(length($"gfw_planted_forests__type"))
+        .cast("boolean") as "gfw_planted_forests__type",
       max("is__gfw_mining_concessions") as "is__gfw_mining_concessions",
       max("is__gfw_managed_forests") as "is__gfw_managed_forests",
       max(length($"rspo_oil_palm__certification_status"))
@@ -164,7 +164,7 @@ object GladAlertsDF {
       max("is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites",
       max("is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas",
       max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands",
-      max("gfw_plantations__type") as "gfw_plantations__type",
+      max("gfw_planted_forests__type") as "gfw_planted_forests__type",
       max("is__gfw_mining_concessions") as "is__gfw_mining_concessions",
       max("is__gfw_managed_forests") as "is__gfw_managed_forests",
       max("rspo_oil_palm__certification_status") as "rspo_oil_palm__certification_status",

src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala

@@ -19,7 +19,7 @@ object IntegratedAlertsDF {
     "is__birdlife_alliance_for_zero_extinction_sites",
     "is__birdlife_key_biodiversity_areas",
     "is__landmark_indigenous_and_community_lands",
-    "gfw_plantations__type",
+    "gfw_planted_forests__type",
     "is__gfw_mining",
     "is__gfw_managed_forests",
     "rspo_oil_palm__certification_status",
@@ -57,7 +57,7 @@ object IntegratedAlertsDF {
         $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_sites",
         $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas",
         $"data_group.landmark" as "is__landmark_indigenous_and_community_lands",
-        $"data_group.plantations" as "gfw_plantations__type",
+        $"data_group.plantations" as "gfw_planted_forests__type",
         $"data_group.mining" as "is__gfw_mining",
         $"data_group.logging" as "is__gfw_managed_forests",
         $"data_group.rspo" as "rspo_oil_palm__certification_status",
@@ -130,8 +130,8 @@ object IntegratedAlertsDF {
       max("is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites",
       max("is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas",
       max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands",
-      max(length($"gfw_plantations__type"))
-        .cast("boolean") as "gfw_plantations__type",
+      max(length($"gfw_planted_forests__type"))
+        .cast("boolean") as "gfw_planted_forests__type",
       max("is__gfw_mining") as "is__gfw_mining",
       max("is__gfw_managed_forests") as "is__gfw_managed_forests",
       max(length($"rspo_oil_palm__certification_status"))
@@ -168,7 +168,7 @@ object IntegratedAlertsDF {
       max("is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites",
       max("is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas",
       max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands",
-      max("gfw_plantations__type") as "gfw_plantations__type",
+      max("gfw_planted_forests__type") as "gfw_planted_forests__type",
       max("is__gfw_mining") as "is__gfw_mining",
       max("is__gfw_managed_forests") as "is__gfw_managed_forests",
       max("rspo_oil_palm__certification_status") as "rspo_oil_palm__certification_status",