diff --git a/terraso_backend/apps/core/gis/mapbox.py b/terraso_backend/apps/core/gis/mapbox.py index 9457aa6f4..db3066844 100644 --- a/terraso_backend/apps/core/gis/mapbox.py +++ b/terraso_backend/apps/core/gis/mapbox.py @@ -84,11 +84,15 @@ def get_publish_status(id): return len(response_json) > 0 +def get_line_delimited_geojson(geojson): + return "\n".join([json.dumps(feature) for feature in geojson["features"]]) + + def _post_tileset_source(geojson, id): - line_delimited_geojson = "\n".join([json.dumps(feature) for feature in geojson["features"]]) + line_delimited_geojson = get_line_delimited_geojson(geojson) url = f"{API_URL}/tilesets/v1/sources/{USERNAME}/{id}?access_token={TOKEN}" - multipart_data = [("file", ("test.ndjson", line_delimited_geojson, "text/plain"))] + multipart_data = [("file", ("input.ndjson", line_delimited_geojson, "text/plain"))] response = requests.post(url, files=multipart_data) response_json = response.json() diff --git a/terraso_backend/apps/shared_data/visualization_tileset_tasks.py b/terraso_backend/apps/shared_data/visualization_tileset_tasks.py index a3d0b55a4..ecd439c34 100644 --- a/terraso_backend/apps/shared_data/visualization_tileset_tasks.py +++ b/terraso_backend/apps/shared_data/visualization_tileset_tasks.py @@ -7,6 +7,7 @@ from django.conf import settings from apps.core.gis.mapbox import create_tileset, remove_tileset +from apps.core.gis.parsers import parse_file_to_geojson from apps.shared_data.services import data_entry_upload_service from .models import VisualizationConfig @@ -62,85 +63,105 @@ def get_owner_name(visualization): return visualization.owner.name if visualization.owner else "Unknown" -def create_mapbox_tileset(visualization_id): - logger.info("Creating mapbox tileset", visualization_id=visualization_id) - visualization = VisualizationConfig.objects.get(pk=visualization_id) - data_entry = visualization.data_entry - owner_name = get_owner_name(visualization) +def _get_geojson_from_dataset(data_entry, visualization): + rows = get_rows_from_file(data_entry) - # You cannot update a Mapbox tileset. We have to delete it and create a new one. - remove_mapbox_tileset(visualization.mapbox_tileset_id) + first_row = rows[0] - try: - rows = get_rows_from_file(data_entry) + dataset_config = visualization.configuration["datasetConfig"] + annotate_config = visualization.configuration["annotateConfig"] - first_row = rows[0] + longitude_column = dataset_config["longitude"] + longitude_index = first_row.index(longitude_column) - dataset_config = visualization.configuration["datasetConfig"] - annotate_config = visualization.configuration["annotateConfig"] + latitude_column = dataset_config["latitude"] + latitude_index = first_row.index(latitude_column) - longitude_column = dataset_config["longitude"] - longitude_index = first_row.index(longitude_column) + data_points = annotate_config["dataPoints"] + data_points_indexes = [ + { + "label": data_point.get("label", data_point["column"]), + "index": first_row.index(data_point["column"]), + } + for data_point in data_points + ] - latitude_column = dataset_config["latitude"] - latitude_index = first_row.index(latitude_column) + annotation_title = annotate_config.get("annotationTitle") - data_points = annotate_config["dataPoints"] - data_points_indexes = [ + title_index = ( + first_row.index(annotation_title) + if annotation_title and annotation_title in first_row + else None + ) + + features = [] + for row in rows: + fields = [ { - "label": data_point.get("label", data_point["column"]), - "index": first_row.index(data_point["column"]), + "label": data_point["label"], + "value": row[data_point["index"]], } - for data_point in data_points + for data_point in data_points_indexes ] - annotation_title = annotate_config.get("annotationTitle") - - title_index = ( - first_row.index(annotation_title) - if annotation_title and annotation_title in first_row - else None - ) + properties = { + "title": row[title_index] if title_index else None, + "fields": json.dumps(fields), + } - features = [] - for row in rows: - fields = [ - { - "label": data_point["label"], - "value": row[data_point["index"]], - } - for data_point in data_points_indexes - ] - - properties = { - "title": row[title_index] if title_index else None, - "fields": json.dumps(fields), + try: + longitude = float(row[longitude_index]) + latitude = float(row[latitude_index]) + feature = { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [longitude, latitude], + }, + "properties": properties, } - try: - longitude = float(row[longitude_index]) - latitude = float(row[latitude_index]) - feature = { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [longitude, latitude], - }, - "properties": properties, - } - - features.append(feature) - except ValueError: - continue - - geojson = { - "type": "FeatureCollection", - "features": features, - } + features.append(feature) + except ValueError: + continue + + return { + "type": "FeatureCollection", + "features": features, + } + + +def _get_geojson_from_gis(data_entry): + file = data_entry_upload_service.get_file(data_entry.s3_object_name, "rt") + return parse_file_to_geojson(file) + + +def _get_geojson_from_data_entry(data_entry, visualization): + is_dataset = f".{data_entry.resource_type}" in settings.DATA_ENTRY_SPREADSHEET_TYPES.keys() + is_gis = f".{data_entry.resource_type}" in settings.DATA_ENTRY_GIS_TYPES.keys() + + if is_dataset: + return _get_geojson_from_dataset(data_entry, visualization) + + if is_gis: + return _get_geojson_from_gis(data_entry) + + +def create_mapbox_tileset(visualization_id): + logger.info("Creating mapbox tileset", visualization_id=visualization_id) + visualization = VisualizationConfig.objects.get(pk=visualization_id) + data_entry = visualization.data_entry + owner_name = get_owner_name(visualization) + + # You cannot update a Mapbox tileset. We have to delete it and create a new one. + remove_mapbox_tileset(visualization.mapbox_tileset_id) + + try: + geojson = _get_geojson_from_data_entry(data_entry, visualization) logger.info( "Geojson generated for mapbox tileset", visualization_id=visualization_id, - rows=len(rows), + features=len(geojson["features"]), ) # Include the environment in the title and description when calling the Mapbox API. diff --git a/terraso_backend/tests/graphql/test_shared_data.py b/terraso_backend/tests/graphql/test_shared_data.py index 644d4e75c..059a54126 100644 --- a/terraso_backend/tests/graphql/test_shared_data.py +++ b/terraso_backend/tests/graphql/test_shared_data.py @@ -297,6 +297,7 @@ def test_data_entries_from_parent_query_by_resource_field(client_query, data_ent for data_entry in data_entries: assert data_entry.name in entries_result + @pytest.fixture def kml_file(request): kml_contents, file_extension = request.param diff --git a/terraso_backend/tests/shared_data/conftest.py b/terraso_backend/tests/shared_data/conftest.py index 407daf294..910409234 100644 --- a/terraso_backend/tests/shared_data/conftest.py +++ b/terraso_backend/tests/shared_data/conftest.py @@ -85,3 +85,19 @@ def visualization_config_b(user_b, data_entry): data_entry=data_entry, created_by=user_b, ) + + +@pytest.fixture +def visualization_config_kml(user): + return mixer.blend( + VisualizationConfig, + size=1, + data_entry=mixer.blend( + DataEntry, + size=1, + url=f"{settings.DATA_ENTRY_FILE_BASE_URL}/{user.id}/test_data.kml", + created_by=user, + resource_type="kml", + ), + created_by=user, + ) diff --git a/terraso_backend/tests/shared_data/test_visualization_tileset_tasks.py b/terraso_backend/tests/shared_data/test_visualization_tileset_tasks.py index ed2afde79..299350be4 100644 --- a/terraso_backend/tests/shared_data/test_visualization_tileset_tasks.py +++ b/terraso_backend/tests/shared_data/test_visualization_tileset_tasks.py @@ -14,17 +14,23 @@ # along with this program. If not, see https://www.gnu.org/licenses/. import io +import json +import os +import tempfile from unittest.mock import patch import pytest import requests +from apps.core.gis.mapbox import get_line_delimited_geojson from apps.shared_data.models import VisualizationConfig from apps.shared_data.visualization_tileset_tasks import ( create_mapbox_tileset, remove_mapbox_tileset, ) +from ..core.gis.test_parsers import KML_CONTENT, KML_GEOJSON + pytestmark = pytest.mark.django_db @@ -37,7 +43,9 @@ def create_mock_response(response): @patch("apps.shared_data.visualization_tileset_tasks.data_entry_upload_service.get_file") @patch("apps.core.gis.mapbox.requests.post") -def test_create_mapbox_tileset_success(mock_request_post, mock_get_file, visualization_config): +def test_create_mapbox_tileset_dataset_success( + mock_request_post, mock_get_file, visualization_config +): visualization_config.configuration = { "datasetConfig": { "longitude": "lng", @@ -53,7 +61,9 @@ def test_create_mapbox_tileset_success(mock_request_post, mock_get_file, visuali }, } visualization_config.save() - mock_get_file.return_value = io.StringIO("lat,lng,col1\nval1,val2,val3") + mock_get_file.return_value = io.StringIO( + "lat,lng,col1\n-78.48306234911033,-0.1805502450716432,val3" + ) mock_responses = [ {"status_code": 200, "json_data": {"id": "tileset-id-1"}}, {"status_code": 200, "json_data": {}}, @@ -64,8 +74,63 @@ def test_create_mapbox_tileset_success(mock_request_post, mock_get_file, visuali updated_visualization_config = VisualizationConfig.objects.get(id=visualization_config.id) assert updated_visualization_config.mapbox_tileset_id is not None assert mock_request_post.call_count == 3 - # Assert post called with params - assert mock_request_post.call_args_list[0][1]["files"] == {} + + assert json.loads(mock_request_post.call_args_list[0][1]["files"][0][1][1]) == { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [-0.1805502450716432, -78.48306234911033]}, + "properties": { + "title": None, + "fields": '[{"label": "label", "value": "val3"}]', + }, + } + + assert ( + mock_request_post.call_args_list[1][1]["json"]["name"] + == f"development - {visualization_config.title}"[:64] + ) + + +@pytest.fixture +def kml_file(): + kml_contents = KML_CONTENT + file_extension = "kml" + with tempfile.NamedTemporaryFile(mode="w", suffix=f".{file_extension}", delete=False) as f: + f.write(kml_contents) + + yield f.name + + os.unlink(f.name) + + +@patch("apps.shared_data.services.data_entry_upload_service.get_file") +@patch("apps.core.gis.mapbox.requests.post") +def test_create_mapbox_tileset_gis_dataentry_success( + mock_request_post, mock_get_file, visualization_config_kml, kml_file +): + with open(kml_file, "rb") as file: + mock_get_file.return_value = file + mock_responses = [ + {"status_code": 200, "json_data": {"id": "tileset-id-1"}}, + {"status_code": 200, "json_data": {}}, + {"status_code": 200, "json_data": {}}, + ] + mock_request_post.side_effect = [ + create_mock_response(response) for response in mock_responses + ] + create_mapbox_tileset(visualization_config_kml.id) + + updated_visualization_config = VisualizationConfig.objects.get(id=visualization_config_kml.id) + assert updated_visualization_config.mapbox_tileset_id is not None + assert mock_request_post.call_count == 3 + + assert mock_request_post.call_args_list[0][1]["files"][0][1][1] == get_line_delimited_geojson( + KML_GEOJSON + ) + + assert ( + mock_request_post.call_args_list[1][1]["json"]["name"] + == f"development - {visualization_config_kml.title}"[:64] + ) @patch("apps.shared_data.visualization_tileset_tasks.data_entry_upload_service.get_file")