Merge pull request #192 from wri/bugfix/afi_ignore_certain_ids

GTC-2570 Filter out rows not requiring analysis
wri · Sep 27, 2023 · ab4e45b · ab4e45b
2 parents 995cd81 + d9391b7
commit ab4e45b
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 3 deletions.
diff --git a/src/main/scala/org/globalforestwatch/summarystats/afi/AFiAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/afi/AFiAnalysis.scala
@@ -48,6 +48,7 @@ object AFiAnalysis extends SummaryAnalysis {
     val summaryDF = AFiAnalysis.aggregateResults(
         AFiDF
         .getFeatureDataFrame(summaryRDD, spark)
+        .filter(!$"gadm_id".contains("null"))
         .withColumn(
           "gadm_id", when(col("location_id") =!= -1|| col("gadm_id").contains("null"), lit("") ).otherwise(col("gadm_id"))
         )

diff --git a/src/main/scala/org/globalforestwatch/summarystats/afi/AFiCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/afi/AFiCommand.scala
@@ -9,13 +9,15 @@ import org.apache.sedona.core.spatialRDD.SpatialRDD
 import org.globalforestwatch.config.GfwConfig
 import org.globalforestwatch.features._
 import org.locationtech.jts.geom.Geometry
+import cats.data.Validated.Valid
+
 
 object AFiCommand extends SummaryCommand {
 
   val afiCommand: Opts[Unit] = Opts.subcommand(
     name = AFiAnalysis.name,
     help = "Compute summary statistics for GFW Pro Dashboard."
-  ) {
+  ) (
     (
       defaultOptions,
       featureFilterOptions,
@@ -30,14 +32,18 @@ object AFiCommand extends SummaryCommand {
 
       runAnalysis { implicit spark =>
         val featureRDD = ValidatedFeatureRDD(default.featureUris, default.featureType, featureFilter, default.splitFeatures)
+        val filteredFeatureRDD = featureRDD.filter{
+          case Valid((GfwProFeatureId(_, locationId), _)) => locationId != -2
+          case _ => true
+        }
 
         AFiAnalysis(
-          featureRDD,
+          filteredFeatureRDD,
           default.featureType,
           spark,
           kwargs
         )
       }
     }
-  }
+  )
 }