Merge pull request #65 from wri/develop

AnnualUpdate - filter out no change rows when exporting change table
wri · Mar 27, 2020 · dcdb289 · dcdb289
2 parents 2027a0b + b77be15
commit dcdb289
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/...a/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala b/...a/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala
@@ -234,6 +234,7 @@ object AnnualUpdateMinimalExport extends SummaryExport {
         .csv(path = outputUrl + "/wdpa/summary")
     }
     exportDF
+      .filter($"treecover_loss__year".isNotNull && $"treecover_loss__ha" > 0)
       .transform(AnnualUpdateMinimalDF.aggChange(idCols, wdpa = true))
       .coalesce(50) // this should result in an avg file size of 100MB
       .write
@@ -272,14 +273,15 @@ object AnnualUpdateMinimalExport extends SummaryExport {
 
       exportDF
         .transform(AnnualUpdateMinimalDF.aggSummary(idCols))
-        .coalesce(4) // this should result in an avg file size of 100MB
+        .coalesce(33) // this should result in an avg file size of 100MB
         .write
         .options(csvOptions)
         .csv(path = outputUrl + "/geostore/summary")
     }
     exportDF
+      .filter($"treecover_loss__year".isNotNull && $"treecover_loss__ha" > 0)
       .transform(AnnualUpdateMinimalDF.aggChange(idCols))
-      .coalesce(10) // this should result in an avg file size of 100MB
+      .coalesce(50) // this should result in an avg file size of 100MB
       .write
       .options(csvOptions)
       .csv(path = outputUrl + "/geostore/change")

diff --git a/version.sbt b/version.sbt
@@ -1 +1 @@
-version in ThisBuild := "1.0.2"
+version in ThisBuild := "1.0.3"
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		version in ThisBuild := "1.0.2"
		version in ThisBuild := "1.0.3"