Skip to content

Commit

Permalink
SW-5880 Added observation aggregates to subzones (#2538)
Browse files Browse the repository at this point in the history 
Implemented logic to fetch observation summary. Some changes as follow:
1) Added subzone stats in an observation by aggregating monitoring plot
2) Method to fetch latest observation per subzone
3) Methods to build up observation summary at zone/site level from the bottoms up. 
4) Added test cases for summary calculations: https://docs.google.com/spreadsheets/d/17-9VlElRgnXU7rnCPsesw6bM5c1Du7Z_ZIvSq8yJPbI/edit?usp=sharing (blue highlighted rows under plot stats reference an older observation)
5) Added observations CSV as a way to bulk import plants within an observation
  • Loading branch information
@tommylau523
tommylau523 authored Nov 4, 2024
1 parent a5f41cc commit a39c5fa
Show file tree
Hide file tree
Showing 13 changed files with 842 additions and 73 deletions.
122 changes: 122 additions & 0 deletions src/main/kotlin/com/terraformation/backend/tracking/api/ObservationsController.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,10 @@ import com.terraformation.backend.tracking.model.ObservationMonitoringPlotResult
import com.terraformation.backend.tracking.model.ObservationMonitoringPlotStatus
import com.terraformation.backend.tracking.model.ObservationPlantingSubzoneResultsModel
import com.terraformation.backend.tracking.model.ObservationPlantingZoneResultsModel
import com.terraformation.backend.tracking.model.ObservationPlantingZoneRollupResultsModel
import com.terraformation.backend.tracking.model.ObservationPlotCounts
import com.terraformation.backend.tracking.model.ObservationResultsModel
import com.terraformation.backend.tracking.model.ObservationRollupResultsModel
import com.terraformation.backend.tracking.model.ObservationSpeciesResultsModel
import com.terraformation.backend.tracking.model.ObservedPlotCoordinatesModel
import com.terraformation.backend.tracking.model.PlantingSiteDepth
Expand Down Expand Up @@ -170,6 +172,16 @@ class ObservationsController(
return ListObservationResultsResponsePayload(results.map { ObservationResultsPayload(it) })
}

@GetMapping("/results/summary")
@Operation(summary = "Gets the rollup observation summary of a planting site")
fun getPlantingSiteObservationSummary(
@RequestParam plantingSiteId: PlantingSiteId,
): GetPlantingSiteObservationSummaryPayload {
val model = observationResultsStore.fetchSummaryForPlantingSite(plantingSiteId)
return GetPlantingSiteObservationSummaryPayload(
model?.let { PlantingSiteObservationSummaryPayload(model) })
}

@GetMapping("/{observationId}")
@Operation(summary = "Gets information about a single observation.")
fun getObservation(@PathVariable observationId: ObservationId): GetObservationResponsePayload {
Expand Down Expand Up @@ -649,14 +661,39 @@ data class ObservationMonitoringPlotResultsPayload(
}

data class ObservationPlantingSubzoneResultsPayload(
@Schema(description = "Area of this planting subzone in hectares.") //
val areaHa: BigDecimal,
val completedTime: Instant?,
@Schema(
description =
"Estimated number of plants in planting subzone based on estimated planting density " +
"and subzone area. Only present if the subzone has completed planting.")
val estimatedPlants: Int?,
@Schema(
description =
"Percentage of plants of all species that were dead in this subzone's permanent " +
"monitoring plots.")
val monitoringPlots: List<ObservationMonitoringPlotResultsPayload>,
val mortalityRate: Int,
@Schema(
description =
"Estimated planting density for the subzone based on the observed planting densities " +
"of monitoring plots. Only present if the subzone has completed planting.")
val plantingDensity: Int?,
val plantingSubzoneId: PlantingSubzoneId,
val totalPlants: Int,
) {
constructor(
model: ObservationPlantingSubzoneResultsModel
) : this(
areaHa = model.areaHa,
completedTime = model.completedTime,
estimatedPlants = model.estimatedPlants,
monitoringPlots = model.monitoringPlots.map { ObservationMonitoringPlotResultsPayload(it) },
mortalityRate = model.mortalityRate,
plantingDensity = model.plantingDensity,
plantingSubzoneId = model.plantingSubzoneId,
totalPlants = model.totalPlants,
)
}

Expand Down Expand Up @@ -763,6 +800,84 @@ data class ObservationResultsPayload(
)
}

data class PlantingZoneObservationSummaryPayload(
@Schema(description = "Area of this planting zone in hectares.") //
val areaHa: BigDecimal,
@Schema(description = "The earliest time of the observations used in this summary.")
val earliestObservationTime: Instant,
@Schema(
description =
"Estimated number of plants in planting zone based on estimated planting density and " +
"planting zone area. Only present if all the subzones in the zone have been " +
"marked as having completed planting.")
val estimatedPlants: Int?,
@Schema(description = "The latest time of the observations used in this summary.")
val latestObservationTime: Instant,
@Schema(
description =
"Percentage of plants of all species that were dead in this zone's permanent " +
"monitoring plots.")
val mortalityRate: Int,
@Schema(
description =
"Estimated planting density for the zone based on the observed planting densities " +
"of monitoring plots. Only present if all the subzones in the zone have been " +
"marked as having completed planting.")
val plantingDensity: Int?,
@Schema(description = "List of subzone observations used in this summary.")
val plantingSubzones: List<ObservationPlantingSubzoneResultsPayload>,
val plantingZoneId: PlantingZoneId,
) {
constructor(
model: ObservationPlantingZoneRollupResultsModel
) : this(
areaHa = model.areaHa,
earliestObservationTime = model.earliestCompletedTime,
estimatedPlants = model.estimatedPlants,
latestObservationTime = model.latestCompletedTime,
mortalityRate = model.mortalityRate,
plantingDensity = model.plantingDensity,
plantingSubzones =
model.plantingSubzones.map { ObservationPlantingSubzoneResultsPayload(it) },
plantingZoneId = model.plantingZoneId,
)
}

data class PlantingSiteObservationSummaryPayload(
@Schema(description = "The earliest time of the observations used in this summary.")
val earliestObservationTime: Instant,
@Schema(
description =
"Estimated total number of live plants at the site, based on the estimated planting " +
"density and site size. Only present if all the subzones in the site have been " +
"marked as having completed planting.")
val estimatedPlants: Int?,
@Schema(description = "The latest time of the observations used in this summary.")
val latestObservationTime: Instant,
@Schema(
description =
"Percentage of plants of all species that were dead in this site's permanent " +
"monitoring plots.")
val mortalityRate: Int?,
@Schema(
description =
"Estimated planting density for the site, based on the observed planting densities " +
"of monitoring plots. Only present if all the subzones in the site have been " +
"marked as having completed planting.")
val plantingDensity: Int?,
val plantingZones: List<PlantingZoneObservationSummaryPayload>
) {
constructor(
model: ObservationRollupResultsModel
) : this(
earliestObservationTime = model.earliestCompletedTime,
estimatedPlants = model.estimatedPlants,
latestObservationTime = model.latestCompletedTime,
mortalityRate = model.mortalityRate,
plantingDensity = model.plantingDensity,
plantingZones = model.plantingZones.map { PlantingZoneObservationSummaryPayload(it) })
}

data class CompletePlotObservationRequestPayload(
val conditions: Set<ObservableCondition>,
val notes: String?,
Expand Down Expand Up @@ -815,6 +930,13 @@ data class ListObservationResultsResponsePayload(
val observations: List<ObservationResultsPayload>
) : SuccessResponsePayload

data class GetPlantingSiteObservationSummaryPayload(
@Schema(
description =
"Rollup summary of planting site observations. Null if no observation has been made.")
val summary: PlantingSiteObservationSummaryPayload?,
) : SuccessResponsePayload

data class ScheduleObservationRequestPayload(
@Schema(
description =
Expand Down
143 changes: 115 additions & 28 deletions src/main/kotlin/com/terraformation/backend/tracking/db/ObservationResultsStore.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,12 @@ import com.terraformation.backend.tracking.model.ObservationMonitoringPlotResult
import com.terraformation.backend.tracking.model.ObservationMonitoringPlotStatus
import com.terraformation.backend.tracking.model.ObservationPlantingSubzoneResultsModel
import com.terraformation.backend.tracking.model.ObservationPlantingZoneResultsModel
import com.terraformation.backend.tracking.model.ObservationPlantingZoneRollupResultsModel
import com.terraformation.backend.tracking.model.ObservationResultsModel
import com.terraformation.backend.tracking.model.ObservationRollupResultsModel
import com.terraformation.backend.tracking.model.ObservationSpeciesResultsModel
import com.terraformation.backend.tracking.model.ObservedPlotCoordinatesModel
import com.terraformation.backend.tracking.model.calculateMortalityRate
import com.terraformation.backend.util.SQUARE_METERS_PER_HECTARE
import jakarta.inject.Named
import kotlin.math.roundToInt
Expand Down Expand Up @@ -73,6 +76,50 @@ class ObservationResultsStore(private val dslContext: DSLContext) {
return fetchByCondition(OBSERVATIONS.plantingSites.ORGANIZATION_ID.eq(organizationId), limit)
}

fun fetchSummaryForPlantingSite(
plantingSiteId: PlantingSiteId,
): ObservationRollupResultsModel? {
val allSubzoneIdsByZoneIds =
dslContext
.select(PLANTING_SUBZONES.ID, PLANTING_SUBZONES.PLANTING_ZONE_ID)
.from(PLANTING_SUBZONES)
.where(PLANTING_SUBZONES.PLANTING_SITE_ID.eq(plantingSiteId))
.groupBy({ it[PLANTING_SUBZONES.PLANTING_ZONE_ID]!! }, { it[PLANTING_SUBZONES.ID]!! })

val zoneAreasById =
dslContext
.select(PLANTING_ZONES.ID, PLANTING_ZONES.AREA_HA)
.from(PLANTING_ZONES)
.where(PLANTING_ZONES.ID.`in`(allSubzoneIdsByZoneIds.keys))
.associate { it[PLANTING_ZONES.ID]!! to it[PLANTING_ZONES.AREA_HA]!! }

val observations = fetchByPlantingSiteId(plantingSiteId)
val subzoneCompletedObservations =
observations
.filter { it.completedTime != null }
.flatMap { observation -> observation.plantingZones.flatMap { it.plantingSubzones } }
.groupBy { it.plantingSubzoneId }

val latestPerSubzone =
subzoneCompletedObservations.mapValues { entry -> entry.value.maxBy { it.completedTime!! } }

val plantingZoneResults =
allSubzoneIdsByZoneIds
.map {
val zoneId = it.key

val areaHa = zoneAreasById[zoneId]!!
val subzoneIds = it.value
val subzoneResults =
subzoneIds.associateWith { subzoneId -> latestPerSubzone[subzoneId] }

zoneId to ObservationPlantingZoneRollupResultsModel.of(areaHa, zoneId, subzoneResults)
}
.toMap()

return ObservationRollupResultsModel.of(plantingSiteId, plantingZoneResults)
}

private val coordinatesGpsField = OBSERVED_PLOT_COORDINATES.GPS_COORDINATES.forMultiset()

private val coordinatesMultiset =
Expand Down Expand Up @@ -256,7 +303,7 @@ class ObservationResultsStore(private val dslContext: DSLContext) {
(it.totalLive + it.totalExisting) > 0
}

val mortalityRate = if (isPermanent) calculateMortalityRate(species) else null
val mortalityRate = if (isPermanent) species.calculateMortalityRate() else null

val areaSquareMeters = sizeMeters * sizeMeters
val plantingDensity =
Expand Down Expand Up @@ -297,7 +344,11 @@ class ObservationResultsStore(private val dslContext: DSLContext) {

private val plantingSubzoneMultiset =
DSL.multiset(
DSL.select(PLANTING_SUBZONES.ID, monitoringPlotMultiset)
DSL.select(
PLANTING_SUBZONES.ID,
PLANTING_SUBZONES.AREA_HA,
PLANTING_SUBZONES.PLANTING_COMPLETED_TIME,
monitoringPlotMultiset)
.from(PLANTING_SUBZONES)
.where(
PLANTING_SUBZONES.ID.`in`(
Expand All @@ -311,9 +362,53 @@ class ObservationResultsStore(private val dslContext: DSLContext) {
.and(PLANTING_SUBZONES.PLANTING_ZONE_ID.eq(PLANTING_ZONES.ID)))))
.convertFrom { results ->
results.map { record ->
val monitoringPlots = record[monitoringPlotMultiset]

val areaHa = record[PLANTING_SUBZONES.AREA_HA.asNonNullable()]

val species = monitoringPlots.flatMap { it.species }
val totalPlants = species.sumOf { it.totalLive + it.totalDead }

val isCompleted =
monitoringPlots.isNotEmpty() && monitoringPlots.all { it.completedTime != null }
val completedTime =
if (isCompleted) {
monitoringPlots.maxOf { it.completedTime!! }
} else {
null
}

val mortalityRate = species.calculateMortalityRate()

val plantingCompleted = record[PLANTING_SUBZONES.PLANTING_COMPLETED_TIME] != null
val plantingDensity =
if (plantingCompleted) {
val plotDensities = monitoringPlots.map { it.plantingDensity }
if (plotDensities.isNotEmpty()) {
plotDensities.average()
} else {
null
}
} else {
null
}

val estimatedPlants =
if (plantingDensity != null && areaHa != null) {
areaHa.toDouble() * plantingDensity
} else {
null
}
ObservationPlantingSubzoneResultsModel(
areaHa = areaHa,
completedTime = completedTime,
estimatedPlants = estimatedPlants?.roundToInt(),
monitoringPlots = record[monitoringPlotMultiset],
mortalityRate = mortalityRate,
plantingCompleted = plantingCompleted,
plantingDensity = plantingDensity?.roundToInt(),
plantingSubzoneId = record[PLANTING_SUBZONES.ID.asNonNullable()],
totalPlants = totalPlants,
)
}
}
Expand Down Expand Up @@ -393,10 +488,11 @@ class ObservationResultsStore(private val dslContext: DSLContext) {
null
}

val mortalityRate = calculateMortalityRate(species)
val mortalityRate = species.calculateMortalityRate()

val plantingCompleted = record[zonePlantingCompletedField]
val plantingDensity =
if (record[zonePlantingCompletedField]) {
if (plantingCompleted) {
val plotDensities =
subzones.flatMap { subzone ->
subzone.monitoringPlots.map { it.plantingDensity }
Expand All @@ -422,6 +518,7 @@ class ObservationResultsStore(private val dslContext: DSLContext) {
completedTime = completedTime,
estimatedPlants = estimatedPlants?.roundToInt(),
mortalityRate = mortalityRate,
plantingCompleted = plantingCompleted,
plantingDensity = plantingDensity?.roundToInt(),
plantingSubzones = subzones,
plantingZoneId = record[PLANTING_ZONES.ID.asNonNullable()],
Expand Down Expand Up @@ -471,11 +568,10 @@ class ObservationResultsStore(private val dslContext: DSLContext) {
val knownSpecies = species.filter { it.certainty != RecordedSpeciesCertainty.Unknown }
val liveSpecies = knownSpecies.filter { it.totalLive > 0 || it.totalExisting > 0 }

var plantingDensity: Int? = null
var estimatedPlants: Int? = null
val plantingCompleted = zones.isNotEmpty() && zones.all { it.plantingCompleted }

if (zones.isNotEmpty() && zones.all { it.plantingDensity != null }) {
plantingDensity =
val plantingDensity =
if (zones.isNotEmpty() && zones.all { it.plantingDensity != null }) {
zones
.flatMap { zone ->
zone.plantingSubzones.flatMap { subzone ->
Expand All @@ -484,21 +580,27 @@ class ObservationResultsStore(private val dslContext: DSLContext) {
}
.average()
.roundToInt()
}
} else {
null
}

if (zones.isNotEmpty() && zones.all { it.estimatedPlants != null }) {
estimatedPlants = zones.mapNotNull { it.estimatedPlants }.sum()
}
val estimatedPlants =
if (zones.isNotEmpty() && zones.all { it.estimatedPlants != null }) {
zones.mapNotNull { it.estimatedPlants }.sum()
} else {
null
}

val totalSpecies = liveSpecies.size

val mortalityRate = calculateMortalityRate(species)
val mortalityRate = species.calculateMortalityRate()

ObservationResultsModel(
completedTime = record[OBSERVATIONS.COMPLETED_TIME],
estimatedPlants = estimatedPlants?.toInt(),
mortalityRate = mortalityRate,
observationId = record[OBSERVATIONS.ID.asNonNullable()],
plantingCompleted = plantingCompleted,
plantingDensity = plantingDensity,
plantingSiteId = record[OBSERVATIONS.PLANTING_SITE_ID.asNonNullable()],
plantingZones = zones,
Expand All @@ -509,19 +611,4 @@ class ObservationResultsStore(private val dslContext: DSLContext) {
)
}
}

/**
* Calculates the mortality rate across all non-preexisting plants of all species in permanent
* monitoring plots.
*/
private fun calculateMortalityRate(species: List<ObservationSpeciesResultsModel>): Int {
val numNonExistingPlants = species.sumOf { it.permanentLive + it.cumulativeDead }
val numDeadPlants = species.sumOf { it.cumulativeDead }

return if (numNonExistingPlants > 0) {
(numDeadPlants * 100.0 / numNonExistingPlants).roundToInt()
} else {
0
}
}
}
Loading

0 comments on commit a39c5fa

Please sign in to comment.