SW-4418 Aggregate germination and loss rates (#1459)

Return the aggregate germination and loss rates in the summary data for species
and for nursery facilities. This replaces the existing aggregate loss rate
calculation, which was less sophisticated than the current one.

To support calculating the aggregate numbers without having to walk through the
quantity history of every batch, the batches table now stores the numerators and
denominators of the batch-level rate calculations, such that the SQL query that
reads the aggregate stats can simply add them up and divide to get the aggregate
rates.

src/main/kotlin/com/terraformation/backend/nursery/api/SpeciesController.kt

@@ -39,11 +39,12 @@ data class SpeciesSummaryNurseryPayload(
 
 data class SpeciesSummaryPayload(
     val germinatingQuantity: Long,
+    val germinationRate: Int?,
     @Schema(
         description = "Percentage of current and past inventory that was withdrawn due to death.",
         minimum = "0",
         maximum = "100")
-    val lossRate: Int,
+    val lossRate: Int?,
     val notReadyQuantity: Long,
     val nurseries: List<SpeciesSummaryNurseryPayload>,
     val readyQuantity: Long,
@@ -60,6 +61,7 @@ data class SpeciesSummaryPayload(
       summary: SpeciesSummary
   ) : this(
       germinatingQuantity = summary.germinatingQuantity,
+      germinationRate = summary.germinationRate,
       lossRate = summary.lossRate,
       notReadyQuantity = summary.notReadyQuantity,
       nurseries = summary.nurseries.map { SpeciesSummaryNurseryPayload(it) },

src/main/kotlin/com/terraformation/backend/nursery/db/BatchStore.kt

@@ -66,6 +66,7 @@ import java.time.Clock
 import java.time.LocalDate
 import kotlin.math.roundToInt
 import org.jooq.DSLContext
+import org.jooq.Field
 import org.jooq.UpdateSetFirstStep
 import org.jooq.UpdateSetMoreStep
 import org.jooq.impl.DSL
@@ -100,6 +101,20 @@ class BatchStore(
 
   private val log = perClassLogger()
 
+  /** The aggregate germination rate for a group of batches. */
+  private val aggregateGerminationRateField: Field<Double?> =
+      DSL.sum(BATCHES.TOTAL_GERMINATED)
+          .times(100.0)
+          .div(DSL.sum(BATCHES.TOTAL_GERMINATION_CANDIDATES))
+          .cast(Double::class.java)
+
+  /** The aggregate loss rate for a group of batches. */
+  private val aggregateLossRateField: Field<Double?> =
+      DSL.sum(BATCHES.TOTAL_LOST)
+          .times(100.0)
+          .div(DSL.sum(BATCHES.TOTAL_LOSS_CANDIDATES))
+          .cast(Double::class.java)
+
   fun fetchOneById(batchId: BatchId): ExistingBatchModel {
     requirePermissions { readBatch(batchId) }
 
@@ -267,14 +282,12 @@ class BatchStore(
             .filter { it.value1() == WithdrawalPurpose.Dead }
             .sumOf { it.value2().toLong() }
 
-    val totalIncludingWithdrawn = totalWithdrawn + (inventory?.totalQuantity ?: 0)
-    val lossRate =
-        if (totalIncludingWithdrawn > 0) {
-          // Round to nearest integer percentage.
-          (totalDead * 100 + totalIncludingWithdrawn / 2) / totalIncludingWithdrawn
-        } else {
-          0
-        }
+    val (germinationRate: Double?, lossRate: Double?) =
+        dslContext
+            .select(aggregateGerminationRateField, aggregateLossRateField)
+            .from(BATCHES)
+            .where(BATCHES.SPECIES_ID.eq(speciesId))
+            .fetchOne()!!
 
     val nurseries =
         dslContext
@@ -288,7 +301,8 @@ class BatchStore(
 
     return SpeciesSummary(
         germinatingQuantity = inventory?.germinatingQuantity ?: 0,
-        lossRate = lossRate.toInt(),
+        germinationRate = germinationRate?.roundToInt(),
+        lossRate = lossRate?.roundToInt(),
         notReadyQuantity = inventory?.notReadyQuantity ?: 0,
         nurseries = nurseries,
         readyQuantity = inventory?.readyQuantity ?: 0,
@@ -949,13 +963,18 @@ class BatchStore(
             .select(
                 DSL.sum(BATCHES.GERMINATING_QUANTITY),
                 DSL.sum(BATCHES.NOT_READY_QUANTITY),
-                DSL.sum(BATCHES.READY_QUANTITY))
+                DSL.sum(BATCHES.READY_QUANTITY),
+                aggregateGerminationRateField,
+                aggregateLossRateField,
+            )
             .from(BATCHES)
             .where(BATCHES.FACILITY_ID.eq(facilityId))
             .fetchOne()
 
     return NurseryStats(
         facilityId = facilityId,
+        germinationRate = inventoryTotals?.value4()?.roundToInt(),
+        lossRate = inventoryTotals?.value5()?.roundToInt(),
         totalGerminating = inventoryTotals?.value1()?.toLong() ?: 0L,
         totalNotReady = inventoryTotals?.value2()?.toLong() ?: 0L,
         totalReady = inventoryTotals?.value3()?.toLong() ?: 0L,
@@ -1115,39 +1134,43 @@ class BatchStore(
         latestEvent[BATCH_QUANTITY_HISTORY.NOT_READY_QUANTITY]!! +
             latestEvent[BATCH_QUANTITY_HISTORY.READY_QUANTITY]!!
 
+    val totalGerminated =
+        currentNotReadyAndReady + totalWithdrawnNotReadyAndReady - initialNotReady - initialReady
+    val totalGerminationCandidates = initialGerminating - totalNonDeadGerminating
     val germinationRate: Int? =
         if (initialGerminating > 0 &&
             currentGerminating == 0 &&
             !hasManualGerminatingEdit &&
             !hasManualNotReadyEdit &&
             !hasManualReadyEdit &&
             !hasAdditionalIncomingTransfers) {
-          val numerator =
-              currentNotReadyAndReady + totalWithdrawnNotReadyAndReady -
-                  initialNotReady -
-                  initialReady
-          val denominator = initialGerminating - totalNonDeadGerminating
-
-          (100.0 * numerator / denominator).roundToInt()
+          (100.0 * totalGerminated / totalGerminationCandidates).roundToInt()
         } else {
           null
         }
 
-    val lossRateDenominator = totalOutplantAndDeadNotReadyAndReady + currentNotReadyAndReady
+    val totalLost = totalDeadNotReadyAndReady
+    val totalLossCandidates = totalOutplantAndDeadNotReadyAndReady + currentNotReadyAndReady
     val lossRate: Int? =
-        if (lossRateDenominator > 0 &&
+        if (totalLossCandidates > 0 &&
             !hasManualNotReadyEdit &&
             !hasManualReadyEdit &&
             !hasAdditionalIncomingTransfers) {
-          (100.0 * totalDeadNotReadyAndReady / lossRateDenominator).roundToInt()
+          (100.0 * totalLost / totalLossCandidates).roundToInt()
         } else {
           null
         }
 
     dslContext
         .update(BATCHES)
         .set(BATCHES.GERMINATION_RATE, germinationRate)
+        .set(BATCHES.TOTAL_GERMINATED, germinationRate?.let { totalGerminated })
+        .set(
+            BATCHES.TOTAL_GERMINATION_CANDIDATES,
+            germinationRate?.let { totalGerminationCandidates })
         .set(BATCHES.LOSS_RATE, lossRate)
+        .set(BATCHES.TOTAL_LOST, lossRate?.let { totalLost })
+        .set(BATCHES.TOTAL_LOSS_CANDIDATES, lossRate?.let { totalLossCandidates })
         .where(BATCHES.ID.eq(batchId))
         .execute()
   }

src/main/kotlin/com/terraformation/backend/nursery/model/NurseryStats.kt

@@ -7,6 +7,8 @@ import kotlin.math.roundToInt
 /** Aggregated statistics for a nursery. Totals are across all batches and withdrawals. */
 data class NurseryStats(
     val facilityId: FacilityId,
+    val germinationRate: Int?,
+    val lossRate: Int?,
     val totalGerminating: Long,
     val totalNotReady: Long,
     val totalReady: Long,

src/main/kotlin/com/terraformation/backend/nursery/model/SpeciesSummary.kt

@@ -5,7 +5,8 @@ import com.terraformation.backend.db.default_schema.tables.pojos.FacilitiesRow
 
 data class SpeciesSummary(
     val germinatingQuantity: Long,
-    val lossRate: Int,
+    val germinationRate: Int?,
+    val lossRate: Int?,
     val notReadyQuantity: Long,
     val nurseries: List<FacilitiesRow>,
     val readyQuantity: Long,

src/main/resources/db/migration/0200/V224__BatchRateTotals.sql

@@ -0,0 +1,4 @@
+ALTER TABLE nursery.batches ADD COLUMN total_germinated INTEGER;
+ALTER TABLE nursery.batches ADD COLUMN total_germination_candidates INTEGER;
+ALTER TABLE nursery.batches ADD COLUMN total_lost INTEGER;
+ALTER TABLE nursery.batches ADD COLUMN total_loss_candidates INTEGER;

src/main/resources/db/migration/R__Comments.sql

@@ -261,6 +261,10 @@ COMMENT ON COLUMN nursery.batches.organization_id IS 'Which organization owns th
 COMMENT ON COLUMN nursery.batches.ready_by_date IS 'User-supplied estimate of when the batch will be ready for planting.';
 COMMENT ON COLUMN nursery.batches.ready_quantity IS 'Number of ready-for-planting seedlings currently available in inventory. Withdrawals cause this to decrease.';
 COMMENT ON COLUMN nursery.batches.species_id IS 'Species of the batch''s plants. Must be under the same organization as the facility ID (enforced in application code).';
+COMMENT ON COLUMN nursery.batches.total_germinated IS 'Total number of seedlings that have moved from Germinating to Not Ready status over the lifetime of the batch. This is the numerator for the germination rate calculation.';
+COMMENT ON COLUMN nursery.batches.total_germination_candidates IS 'Total number of seedlings that have been candidates for moving from Germinating to Not Ready status. This includes seedlings that are already germinated and germinating seedlings that were withdrawn as Dead, but does not include germinating seedlings that were withdrawn for other reasons. This is the denominator for the germination rate calculation.';
+COMMENT ON COLUMN nursery.batches.total_loss_candidates IS 'Total number of non-germinating (Not Ready and Ready) seedlings that have been candidates for being withdrawn as dead. This includes seedlings that are still in the batch, seedlings that were withdrawn for outplanting, and seedlings that were already withdrawn as dead, but does not include germinating seedlings or seedlings that were withdrawn for other reasons. This is the denominator for the loss rate calculation.';
+COMMENT ON COLUMN nursery.batches.total_lost IS 'Total number of non-germinating (Not Ready and Ready) seedlings that have been withdrawn as Dead. This is the numerator for the loss rate calculation.';
 COMMENT ON COLUMN nursery.batches.version IS 'Increases by 1 each time the batch is modified. Used to detect when clients have stale data about batches.';
 
 COMMENT ON TABLE nursery.withdrawal_photos IS 'Linking table between `withdrawals` and `files`.';

src/test/kotlin/com/terraformation/backend/db/DatabaseTest.kt

@@ -153,6 +153,7 @@ import java.time.Instant
 import java.time.LocalDate
 import java.time.ZoneId
 import java.util.Locale
+import kotlin.math.roundToInt
 import kotlin.reflect.full.createType
 import kotlin.reflect.full.isSupertypeOf
 import org.jooq.Configuration
@@ -834,8 +835,30 @@ abstract class DatabaseTest {
       version: Int = row.version ?: 1,
       batchNumber: String = row.batchNumber ?: id?.toString() ?: "${nextBatchNuber++}",
       germinationRate: Int? = row.germinationRate,
+      totalGerminated: Int? = row.totalGerminated,
+      totalGerminationCandidates: Int? = row.totalGerminationCandidates,
       lossRate: Int? = row.lossRate ?: if (notReadyQuantity > 0 || readyQuantity > 0) 0 else null,
+      totalLost: Int? = row.totalLost ?: if (notReadyQuantity > 0 || readyQuantity > 0) 0 else null,
+      totalLossCandidates: Int? =
+          row.totalLossCandidates
+              ?: if (notReadyQuantity > 0 || readyQuantity > 0) notReadyQuantity + readyQuantity
+              else null,
   ): BatchId {
+    val effectiveGerminationRate =
+        germinationRate
+            ?: if (totalGerminated != null && totalGerminationCandidates != null) {
+              ((100.0 * totalGerminated) / totalGerminationCandidates).roundToInt()
+            } else {
+              null
+            }
+    val effectiveLossRate =
+        lossRate
+            ?: if (totalLost != null && totalLossCandidates != null) {
+              ((100.0 * totalLost) / totalLossCandidates).roundToInt()
+            } else {
+              null
+            }
+
     val rowWithDefaults =
         row.copy(
             addedDate = addedDate,
@@ -844,13 +867,17 @@ abstract class DatabaseTest {
             createdTime = createdTime,
             facilityId = facilityId.toIdWrapper { FacilityId(it) },
             germinatingQuantity = germinatingQuantity,
-            germinationRate = germinationRate,
+            germinationRate = effectiveGerminationRate,
+            totalGerminated = totalGerminated,
+            totalGerminationCandidates = totalGerminationCandidates,
             id = id?.toIdWrapper { BatchId(it) },
             latestObservedGerminatingQuantity = germinatingQuantity,
             latestObservedNotReadyQuantity = notReadyQuantity,
             latestObservedReadyQuantity = readyQuantity,
             latestObservedTime = createdTime,
-            lossRate = lossRate,
+            lossRate = effectiveLossRate,
+            totalLost = totalLost,
+            totalLossCandidates = totalLossCandidates,
             modifiedBy = createdBy,
             modifiedTime = createdTime,
             notReadyQuantity = notReadyQuantity,

src/test/kotlin/com/terraformation/backend/nursery/db/batchStore/BatchStoreChangeStatusesTest.kt

@@ -42,6 +42,9 @@ internal class BatchStoreChangeStatusesTest : BatchStoreTest() {
             germinatingQuantity = 8,
             notReadyQuantity = 15,
             readyQuantity = 37,
+            totalLost = 0,
+            // moved 2 seeds from germinating to not-ready
+            totalLossCandidates = 52,
             modifiedTime = updateTime,
             version = 2),
         after)

src/test/kotlin/com/terraformation/backend/nursery/db/batchStore/BatchStoreDeleteBatchTest.kt

@@ -102,9 +102,28 @@ internal class BatchStoreDeleteBatchTest : BatchStoreTest() {
   @Test
   fun `species summary is updated to reflect deleted batch`() {
     // This batch is not deleted
-    insertBatch(germinatingQuantity = 100, notReadyQuantity = 200, readyQuantity = 300)
-
-    val batchId = insertBatch(germinatingQuantity = 1, notReadyQuantity = 2, readyQuantity = 3)
+    insertBatch(
+        germinatingQuantity = 100,
+        germinationRate = 50,
+        totalGerminationCandidates = 50,
+        totalGerminated = 25,
+        lossRate = 25,
+        totalLossCandidates = 100,
+        totalLost = 25,
+        notReadyQuantity = 200,
+        readyQuantity = 300)
+
+    val batchId =
+        insertBatch(
+            germinatingQuantity = 1,
+            germinationRate = 100,
+            totalGerminationCandidates = 10,
+            totalGerminated = 10,
+            lossRate = 0,
+            totalLossCandidates = 100,
+            totalLost = 0,
+            notReadyQuantity = 2,
+            readyQuantity = 3)
     insertWithdrawal(purpose = WithdrawalPurpose.Dead)
     insertBatchWithdrawal(
         germinatingQuantityWithdrawn = 10,
@@ -114,9 +133,10 @@ internal class BatchStoreDeleteBatchTest : BatchStoreTest() {
     val summaryBeforeDelete =
         SpeciesSummary(
             germinatingQuantity = 101,
+            germinationRate = 58,
             notReadyQuantity = 202,
             readyQuantity = 303,
-            lossRate = 9,
+            lossRate = 13,
             nurseries = listOf(FacilitiesRow(id = facilityId, name = "Nursery")),
             speciesId = speciesId,
             totalDead = 50,
@@ -130,9 +150,10 @@ internal class BatchStoreDeleteBatchTest : BatchStoreTest() {
     assertEquals(
         SpeciesSummary(
             germinatingQuantity = 100,
+            germinationRate = 50,
             notReadyQuantity = 200,
             readyQuantity = 300,
-            lossRate = 0,
+            lossRate = 25,
             nurseries = summaryBeforeDelete.nurseries,
             speciesId = speciesId,
             totalDead = 0,

src/test/kotlin/com/terraformation/backend/nursery/db/batchStore/BatchStoreGetNurseryStatsTest.kt

@@ -23,14 +23,20 @@ internal class BatchStoreGetNurseryStatsTest : BatchStoreTest() {
             speciesId = speciesId,
             germinatingQuantity = 1,
             notReadyQuantity = 2,
-            readyQuantity = 3)
+            readyQuantity = 3,
+            totalGerminated = 100,
+            totalGerminationCandidates = 200,
+            totalLost = 300,
+            totalLossCandidates = 400)
     val batchId2 =
         insertBatch(
             facilityId = facilityId,
             speciesId = speciesId2,
             germinatingQuantity = 4,
             notReadyQuantity = 5,
-            readyQuantity = 6)
+            readyQuantity = 6,
+            totalLost = 500,
+            totalLossCandidates = 600)
 
     insertWithdrawal(facilityId = facilityId, purpose = WithdrawalPurpose.OutPlant)
     insertBatchWithdrawal(
@@ -75,7 +81,11 @@ internal class BatchStoreGetNurseryStatsTest : BatchStoreTest() {
         speciesId = speciesId,
         germinatingQuantity = 7,
         notReadyQuantity = 8,
-        readyQuantity = 9)
+        readyQuantity = 9,
+        totalGerminated = 700,
+        totalGerminationCandidates = 800,
+        totalLost = 900,
+        totalLossCandidates = 1000)
     insertWithdrawal(facilityId = otherNurseryId, purpose = WithdrawalPurpose.OutPlant)
     insertBatchWithdrawal(
         germinatingQuantityWithdrawn = 28,
@@ -91,6 +101,8 @@ internal class BatchStoreGetNurseryStatsTest : BatchStoreTest() {
     val expected =
         NurseryStats(
             facilityId = facilityId,
+            germinationRate = 50,
+            lossRate = 80,
             totalGerminating = 1 + 4,
             totalNotReady = 2 + 5,
             totalReady = 3 + 6,