feat: soil id cache (#1357)

terraso_backend/apps/soil_id/graphql/soil_id/resolvers.py

@@ -13,6 +13,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see https://www.gnu.org/licenses/.
 
+import math
 import traceback
 from typing import Optional
 
@@ -39,6 +40,7 @@
 )
 from apps.soil_id.models.depth_dependent_soil_data import DepthDependentSoilData
 from apps.soil_id.models.soil_data import SoilData
+from apps.soil_id.models.soil_id_cache import SoilIdCache
 
 logger = structlog.get_logger(__name__)
 
@@ -78,9 +80,11 @@ def resolve_soil_data(soil_match) -> SoilIdSoilData:
         )
         prev_depth = bottom_depth
 
-    return SoilIdSoilData(
-        slope=soil_match["site"]["siteData"]["slope"], depth_dependent_data=depth_dependent_data
-    )
+    slope = soil_match["site"]["siteData"]["slope"]
+    if slope == "":
+        slope = None
+
+    return SoilIdSoilData(slope=slope, depth_dependent_data=depth_dependent_data)
 
 
 def resolve_ecological_site(ecological_site: dict):
@@ -142,18 +146,45 @@ def resolve_list_output_failure(list_output: SoilListOutputData | str):
     if isinstance(list_output, SoilListOutputData):
         return None
     elif isinstance(list_output, str):
-        return SoilIdFailure(reason=SoilIdFailureReason.DATA_UNAVAILABLE)
+        return SoilIdFailureReason.DATA_UNAVAILABLE
     else:
-        return SoilIdFailure(reason=SoilIdFailureReason.ALGORITHM_FAILURE)
+        return SoilIdFailureReason.ALGORITHM_FAILURE
+
+
+def clean_soil_list_json(obj):
+    if isinstance(obj, float) and math.isnan(obj):
+        return None
+    elif isinstance(obj, dict):
+        return dict((k, clean_soil_list_json(v)) for k, v in obj.items())
+    elif isinstance(obj, (list, set, tuple)):
+        return list(map(clean_soil_list_json, obj))
+    return obj
+
+
+def get_cached_list_soils_output(latitude, longitude):
+    cached_result = SoilIdCache.get_data(latitude=latitude, longitude=longitude)
+    if cached_result is None:
+        list_output = list_soils(lat=latitude, lon=longitude)
+        failure_reason = resolve_list_output_failure(list_output)
+
+        if failure_reason is not None:
+            list_output = failure_reason.value
+        else:
+            list_output.soil_list_json = clean_soil_list_json(list_output.soil_list_json)
+
+        SoilIdCache.save_data(latitude=latitude, longitude=longitude, data=list_output)
+
+        return list_output
+    else:
+        return cached_result
 
 
 def resolve_location_based_result(_parent, _info, latitude: float, longitude: float):
     try:
-        list_output = list_soils(lat=latitude, lon=longitude)
+        list_output = get_cached_list_soils_output(latitude=latitude, longitude=longitude)
 
-        failure = resolve_list_output_failure(list_output)
-        if failure is not None:
-            return failure
+        if isinstance(list_output, str):
+            return SoilIdFailure(reason=list_output)
 
         return resolve_location_based_soil_matches(list_output.soil_list_json)
     except Exception:
@@ -266,11 +297,10 @@ def resolve_data_based_result(
     _parent, _info, latitude: float, longitude: float, data: SoilIdInputData
 ):
     try:
-        list_output = list_soils(lat=latitude, lon=longitude)
+        list_output = get_cached_list_soils_output(latitude=latitude, longitude=longitude)
 
-        failure = resolve_list_output_failure(list_output)
-        if failure is not None:
-            return failure
+        if isinstance(list_output, str):
+            return SoilIdFailure(reason=list_output)
 
         rank_output = rank_soils(
             lat=latitude,

terraso_backend/apps/soil_id/migrations/0014_soilidcache_soilidcache_coordinate_index.py

@@ -0,0 +1,51 @@
+# Copyright © 2024 Technology Matters
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see https://www.gnu.org/licenses/.
+
+# Generated by Django 5.0.6 on 2024-06-27 23:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("soil_id", "0013_remove_projectsoilsettings_measurement_units"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="SoilIdCache",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("latitude", models.FloatField()),
+                ("longitude", models.FloatField()),
+                ("failure_reason", models.TextField(null=True)),
+                ("soil_list_json", models.JSONField(null=True)),
+                ("rank_data_csv", models.TextField(null=True)),
+                ("map_unit_component_data_csv", models.TextField(null=True)),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name="soilidcache",
+            constraint=models.UniqueConstraint(
+                fields=("latitude", "longitude"), name="coordinate_index"
+            ),
+        ),
+    ]

terraso_backend/apps/soil_id/models/__init__.py

@@ -23,6 +23,7 @@
     ProjectSoilSettings,
 )
 from .soil_data import SoilData, SoilDataDepthInterval
+from .soil_id_cache import SoilIdCache
 
 __all__ = [
     "SoilData",
@@ -33,4 +34,5 @@
     "LandPKSIntervalDefaults",
     "NRCSIntervalDefaults",
     "DepthIntervalPreset",
+    "SoilIdCache",
 ]

terraso_backend/apps/soil_id/models/soil_id_cache.py

@@ -0,0 +1,70 @@
+# Copyright © 2024 Technology Matters
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see https://www.gnu.org/licenses/.
+
+
+from django.db import models
+from soil_id.us_soil import SoilListOutputData
+
+
+class SoilIdCache(models.Model):
+    latitude = models.FloatField()
+    longitude = models.FloatField()
+    failure_reason = models.TextField(null=True)
+    soil_list_json = models.JSONField(null=True)
+    rank_data_csv = models.TextField(null=True)
+    map_unit_component_data_csv = models.TextField(null=True)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(fields=["latitude", "longitude"], name="coordinate_index")
+        ]
+
+    @classmethod
+    def round_coordinate(cls, coord: float):
+        return round(coord, 6)
+
+    @classmethod
+    def save_data(cls, latitude: float, longitude: float, data: SoilListOutputData | str):
+        if isinstance(data, str):
+            data_to_save = {"failure_reason": data}
+        else:
+            data_to_save = {
+                "soil_list_json": data.soil_list_json,
+                "rank_data_csv": data.rank_data_csv,
+                "map_unit_component_data_csv": data.map_unit_component_data_csv,
+            }
+
+        cls.objects.update_or_create(
+            latitude=cls.round_coordinate(latitude),
+            longitude=cls.round_coordinate(longitude),
+            create_defaults=data_to_save,
+        )
+
+    @classmethod
+    def get_data(cls, latitude: float, longitude: float) -> SoilListOutputData | str:
+        try:
+            prev_result = cls.objects.get(
+                latitude=cls.round_coordinate(latitude), longitude=cls.round_coordinate(longitude)
+            )
+            if prev_result.failure_reason is not None:
+                return prev_result.failure_reason
+
+            return SoilListOutputData(
+                soil_list_json=prev_result.soil_list_json,
+                rank_data_csv=prev_result.rank_data_csv,
+                map_unit_component_data_csv=prev_result.map_unit_component_data_csv,
+            )
+        except cls.DoesNotExist:
+            return None