fix: Create mapbox tileset for GIS formats

terraso_backend/apps/core/gis/mapbox.py

@@ -84,11 +84,15 @@ def get_publish_status(id):
     return len(response_json) > 0
 
 
+def get_line_delimited_geojson(geojson):
+    return "\n".join([json.dumps(feature) for feature in geojson["features"]])
+
+
 def _post_tileset_source(geojson, id):
-    line_delimited_geojson = "\n".join([json.dumps(feature) for feature in geojson["features"]])
+    line_delimited_geojson = get_line_delimited_geojson(geojson)
 
     url = f"{API_URL}/tilesets/v1/sources/{USERNAME}/{id}?access_token={TOKEN}"
-    multipart_data = [("file", ("test.ndjson", line_delimited_geojson, "text/plain"))]
+    multipart_data = [("file", ("input.ndjson", line_delimited_geojson, "text/plain"))]
 
     response = requests.post(url, files=multipart_data)
     response_json = response.json()

terraso_backend/apps/shared_data/visualization_tileset_tasks.py

@@ -7,6 +7,7 @@
 from django.conf import settings
 
 from apps.core.gis.mapbox import create_tileset, remove_tileset
+from apps.core.gis.parsers import parse_file_to_geojson
 from apps.shared_data.services import data_entry_upload_service
 
 from .models import VisualizationConfig
@@ -62,85 +63,105 @@ def get_owner_name(visualization):
     return visualization.owner.name if visualization.owner else "Unknown"
 
 
-def create_mapbox_tileset(visualization_id):
-    logger.info("Creating mapbox tileset", visualization_id=visualization_id)
-    visualization = VisualizationConfig.objects.get(pk=visualization_id)
-    data_entry = visualization.data_entry
-    owner_name = get_owner_name(visualization)
+def _get_geojson_from_dataset(data_entry, visualization):
+    rows = get_rows_from_file(data_entry)
 
-    # You cannot update a Mapbox tileset. We have to delete it and create a new one.
-    remove_mapbox_tileset(visualization.mapbox_tileset_id)
+    first_row = rows[0]
 
-    try:
-        rows = get_rows_from_file(data_entry)
+    dataset_config = visualization.configuration["datasetConfig"]
+    annotate_config = visualization.configuration["annotateConfig"]
 
-        first_row = rows[0]
+    longitude_column = dataset_config["longitude"]
+    longitude_index = first_row.index(longitude_column)
 
-        dataset_config = visualization.configuration["datasetConfig"]
-        annotate_config = visualization.configuration["annotateConfig"]
+    latitude_column = dataset_config["latitude"]
+    latitude_index = first_row.index(latitude_column)
 
-        longitude_column = dataset_config["longitude"]
-        longitude_index = first_row.index(longitude_column)
+    data_points = annotate_config["dataPoints"]
+    data_points_indexes = [
+        {
+            "label": data_point.get("label", data_point["column"]),
+            "index": first_row.index(data_point["column"]),
+        }
+        for data_point in data_points
+    ]
 
-        latitude_column = dataset_config["latitude"]
-        latitude_index = first_row.index(latitude_column)
+    annotation_title = annotate_config.get("annotationTitle")
 
-        data_points = annotate_config["dataPoints"]
-        data_points_indexes = [
+    title_index = (
+        first_row.index(annotation_title)
+        if annotation_title and annotation_title in first_row
+        else None
+    )
+
+    features = []
+    for row in rows:
+        fields = [
             {
-                "label": data_point.get("label", data_point["column"]),
-                "index": first_row.index(data_point["column"]),
+                "label": data_point["label"],
+                "value": row[data_point["index"]],
             }
-            for data_point in data_points
+            for data_point in data_points_indexes
         ]
 
-        annotation_title = annotate_config.get("annotationTitle")
-
-        title_index = (
-            first_row.index(annotation_title)
-            if annotation_title and annotation_title in first_row
-            else None
-        )
+        properties = {
+            "title": row[title_index] if title_index else None,
+            "fields": json.dumps(fields),
+        }
 
-        features = []
-        for row in rows:
-            fields = [
-                {
-                    "label": data_point["label"],
-                    "value": row[data_point["index"]],
-                }
-                for data_point in data_points_indexes
-            ]
-
-            properties = {
-                "title": row[title_index] if title_index else None,
-                "fields": json.dumps(fields),
+        try:
+            longitude = float(row[longitude_index])
+            latitude = float(row[latitude_index])
+            feature = {
+                "type": "Feature",
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [longitude, latitude],
+                },
+                "properties": properties,
             }
 
-            try:
-                longitude = float(row[longitude_index])
-                latitude = float(row[latitude_index])
-                feature = {
-                    "type": "Feature",
-                    "geometry": {
-                        "type": "Point",
-                        "coordinates": [longitude, latitude],
-                    },
-                    "properties": properties,
-                }
-
-                features.append(feature)
-            except ValueError:
-                continue
-
-        geojson = {
-            "type": "FeatureCollection",
-            "features": features,
-        }
+            features.append(feature)
+        except ValueError:
+            continue
+
+    return {
+        "type": "FeatureCollection",
+        "features": features,
+    }
+
+
+def _get_geojson_from_gis(data_entry):
+    file = data_entry_upload_service.get_file(data_entry.s3_object_name, "rt")
+    return parse_file_to_geojson(file)
+
+
+def _get_geojson_from_data_entry(data_entry, visualization):
+    is_dataset = f".{data_entry.resource_type}" in settings.DATA_ENTRY_SPREADSHEET_TYPES.keys()
+    is_gis = f".{data_entry.resource_type}" in settings.DATA_ENTRY_GIS_TYPES.keys()
+
+    if is_dataset:
+        return _get_geojson_from_dataset(data_entry, visualization)
+
+    if is_gis:
+        return _get_geojson_from_gis(data_entry)
+
+
+def create_mapbox_tileset(visualization_id):
+    logger.info("Creating mapbox tileset", visualization_id=visualization_id)
+    visualization = VisualizationConfig.objects.get(pk=visualization_id)
+    data_entry = visualization.data_entry
+    owner_name = get_owner_name(visualization)
+
+    # You cannot update a Mapbox tileset. We have to delete it and create a new one.
+    remove_mapbox_tileset(visualization.mapbox_tileset_id)
+
+    try:
+        geojson = _get_geojson_from_data_entry(data_entry, visualization)
         logger.info(
             "Geojson generated for mapbox tileset",
             visualization_id=visualization_id,
-            rows=len(rows),
+            features=len(geojson["features"]),
         )
 
         # Include the environment in the title and description when calling the Mapbox API.

terraso_backend/tests/graphql/test_shared_data.py

@@ -297,6 +297,7 @@ def test_data_entries_from_parent_query_by_resource_field(client_query, data_ent
     for data_entry in data_entries:
         assert data_entry.name in entries_result
 
+
 @pytest.fixture
 def kml_file(request):
     kml_contents, file_extension = request.param

terraso_backend/tests/shared_data/conftest.py

@@ -85,3 +85,19 @@ def visualization_config_b(user_b, data_entry):
         data_entry=data_entry,
         created_by=user_b,
     )
+
+
+@pytest.fixture
+def visualization_config_kml(user):
+    return mixer.blend(
+        VisualizationConfig,
+        size=1,
+        data_entry=mixer.blend(
+            DataEntry,
+            size=1,
+            url=f"{settings.DATA_ENTRY_FILE_BASE_URL}/{user.id}/test_data.kml",
+            created_by=user,
+            resource_type="kml",
+        ),
+        created_by=user,
+    )

terraso_backend/tests/shared_data/test_visualization_tileset_tasks.py

@@ -14,17 +14,23 @@
 # along with this program. If not, see https://www.gnu.org/licenses/.
 
 import io
+import json
+import os
+import tempfile
 from unittest.mock import patch
 
 import pytest
 import requests
 
+from apps.core.gis.mapbox import get_line_delimited_geojson
 from apps.shared_data.models import VisualizationConfig
 from apps.shared_data.visualization_tileset_tasks import (
     create_mapbox_tileset,
     remove_mapbox_tileset,
 )
 
+from ..core.gis.test_parsers import KML_CONTENT, KML_GEOJSON
+
 pytestmark = pytest.mark.django_db
 
 
@@ -37,7 +43,9 @@ def create_mock_response(response):
 
 @patch("apps.shared_data.visualization_tileset_tasks.data_entry_upload_service.get_file")
 @patch("apps.core.gis.mapbox.requests.post")
-def test_create_mapbox_tileset_success(mock_request_post, mock_get_file, visualization_config):
+def test_create_mapbox_tileset_dataset_success(
+    mock_request_post, mock_get_file, visualization_config
+):
     visualization_config.configuration = {
         "datasetConfig": {
             "longitude": "lng",
@@ -53,7 +61,9 @@ def test_create_mapbox_tileset_success(mock_request_post, mock_get_file, visuali
         },
     }
     visualization_config.save()
-    mock_get_file.return_value = io.StringIO("lat,lng,col1\nval1,val2,val3")
+    mock_get_file.return_value = io.StringIO(
+        "lat,lng,col1\n-78.48306234911033,-0.1805502450716432,val3"
+    )
     mock_responses = [
         {"status_code": 200, "json_data": {"id": "tileset-id-1"}},
         {"status_code": 200, "json_data": {}},
@@ -64,8 +74,63 @@ def test_create_mapbox_tileset_success(mock_request_post, mock_get_file, visuali
     updated_visualization_config = VisualizationConfig.objects.get(id=visualization_config.id)
     assert updated_visualization_config.mapbox_tileset_id is not None
     assert mock_request_post.call_count == 3
-    # Assert post called with params
-    assert mock_request_post.call_args_list[0][1]["files"] == {}
+
+    assert json.loads(mock_request_post.call_args_list[0][1]["files"][0][1][1]) == {
+        "type": "Feature",
+        "geometry": {"type": "Point", "coordinates": [-0.1805502450716432, -78.48306234911033]},
+        "properties": {
+            "title": None,
+            "fields": '[{"label": "label", "value": "val3"}]',
+        },
+    }
+
+    assert (
+        mock_request_post.call_args_list[1][1]["json"]["name"]
+        == f"development - {visualization_config.title}"[:64]
+    )
+
+
+@pytest.fixture
+def kml_file():
+    kml_contents = KML_CONTENT
+    file_extension = "kml"
+    with tempfile.NamedTemporaryFile(mode="w", suffix=f".{file_extension}", delete=False) as f:
+        f.write(kml_contents)
+
+    yield f.name
+
+    os.unlink(f.name)
+
+
+@patch("apps.shared_data.services.data_entry_upload_service.get_file")
+@patch("apps.core.gis.mapbox.requests.post")
+def test_create_mapbox_tileset_gis_dataentry_success(
+    mock_request_post, mock_get_file, visualization_config_kml, kml_file
+):
+    with open(kml_file, "rb") as file:
+        mock_get_file.return_value = file
+        mock_responses = [
+            {"status_code": 200, "json_data": {"id": "tileset-id-1"}},
+            {"status_code": 200, "json_data": {}},
+            {"status_code": 200, "json_data": {}},
+        ]
+        mock_request_post.side_effect = [
+            create_mock_response(response) for response in mock_responses
+        ]
+        create_mapbox_tileset(visualization_config_kml.id)
+
+    updated_visualization_config = VisualizationConfig.objects.get(id=visualization_config_kml.id)
+    assert updated_visualization_config.mapbox_tileset_id is not None
+    assert mock_request_post.call_count == 3
+
+    assert mock_request_post.call_args_list[0][1]["files"][0][1][1] == get_line_delimited_geojson(
+        KML_GEOJSON
+    )
+
+    assert (
+        mock_request_post.call_args_list[1][1]["json"]["name"]
+        == f"development - {visualization_config_kml.title}"[:64]
+    )
 
 
 @patch("apps.shared_data.visualization_tileset_tasks.data_entry_upload_service.get_file")