From 2162f0e6d8eaf1e8b02ad50df24e350eb19db0c8 Mon Sep 17 00:00:00 2001 From: Jose Buitron Date: Wed, 8 Nov 2023 15:07:43 -0500 Subject: [PATCH] feat: Visualization (data map) GIS formats support (#917) --- terraso_backend/apps/core/gis/mapbox.py | 8 +- terraso_backend/apps/core/gis/parsers.py | 37 ++++ terraso_backend/apps/core/views.py | 44 +--- .../apps/graphql/schema/data_entries.py | 14 ++ .../apps/graphql/schema/schema.graphql | 3 + .../graphql/schema/visualization_config.py | 2 + .../0017_visualizationconfig_description.py | 35 +++ .../models/visualization_config.py | 1 + .../visualization_tileset_tasks.py | 145 ++++++------ .../tests/core/gis/test_parsers.py | 206 +++++++++--------- terraso_backend/tests/graphql/conftest.py | 30 +++ .../tests/graphql/test_shared_data.py | 120 ++++++++++ .../graphql/test_visualization_config.py | 4 + terraso_backend/tests/shared_data/conftest.py | 16 ++ .../test_visualization_tileset_tasks.py | 71 +++++- 15 files changed, 527 insertions(+), 209 deletions(-) create mode 100644 terraso_backend/apps/shared_data/migrations/0017_visualizationconfig_description.py diff --git a/terraso_backend/apps/core/gis/mapbox.py b/terraso_backend/apps/core/gis/mapbox.py index 9457aa6f4..db3066844 100644 --- a/terraso_backend/apps/core/gis/mapbox.py +++ b/terraso_backend/apps/core/gis/mapbox.py @@ -84,11 +84,15 @@ def get_publish_status(id): return len(response_json) > 0 +def get_line_delimited_geojson(geojson): + return "\n".join([json.dumps(feature) for feature in geojson["features"]]) + + def _post_tileset_source(geojson, id): - line_delimited_geojson = "\n".join([json.dumps(feature) for feature in geojson["features"]]) + line_delimited_geojson = get_line_delimited_geojson(geojson) url = f"{API_URL}/tilesets/v1/sources/{USERNAME}/{id}?access_token={TOKEN}" - multipart_data = [("file", ("test.ndjson", line_delimited_geojson, "text/plain"))] + multipart_data = [("file", ("input.ndjson", line_delimited_geojson, "text/plain"))] response = requests.post(url, files=multipart_data) response_json = response.json() diff --git a/terraso_backend/apps/core/gis/parsers.py b/terraso_backend/apps/core/gis/parsers.py index 5d2d7b429..17e90f5c9 100644 --- a/terraso_backend/apps/core/gis/parsers.py +++ b/terraso_backend/apps/core/gis/parsers.py @@ -19,13 +19,21 @@ import zipfile import geopandas as gpd +import structlog +from django.core.exceptions import ValidationError from fiona.drvsupport import supported_drivers from apps.core.gis.utils import DEFAULT_CRS +logger = structlog.get_logger(__name__) + supported_drivers["KML"] = "rw" +def is_geojson_file_extension(file): + return file.name.endswith((".geojson", ".json")) + + def is_shape_file_extension(file): return file.name.endswith(".zip") @@ -96,3 +104,32 @@ def parse_shapefile(file): os.rmdir(tmp_folder) return json.loads(gdf_transformed.to_json()) + + +def parse_file_to_geojson(file): + if is_shape_file_extension(file): + try: + return parse_shapefile(file) + except Exception as e: + logger.error("Error parsing shapefile", error=e) + raise ValidationError("invalid_shapefile") + elif is_kml_file_extension(file): + try: + return parse_kml_file(file) + except Exception as e: + logger.error("Error parsing kml file", error=e) + raise ValidationError("invalid_kml_file") + elif is_kmz_file_extension(file): + try: + return parse_kmz_file(file) + except Exception as e: + logger.error("Error parsing kmz file", error=e) + raise ValidationError("invalid_kmz_file") + elif is_geojson_file_extension(file): + try: + return json.load(file) + except Exception as e: + logger.error("Error parsing geojson file", error=e) + raise ValidationError("invalid_geojson_file") + else: + raise ValidationError("invalid_file_type") diff --git a/terraso_backend/apps/core/views.py b/terraso_backend/apps/core/views.py index 7a682a339..bbe10a0e1 100644 --- a/terraso_backend/apps/core/views.py +++ b/terraso_backend/apps/core/views.py @@ -25,6 +25,7 @@ from django.contrib.admin.views.decorators import staff_member_required from django.contrib.sessions.models import Session from django.core import management +from django.core.exceptions import ValidationError from django.db import DatabaseError, transaction from django.db.transaction import get_connection from django.http import ( @@ -37,14 +38,7 @@ from django.views.generic.edit import FormView from apps.auth.mixins import AuthenticationRequiredMixin -from apps.core.gis.parsers import ( - is_kml_file_extension, - is_kmz_file_extension, - is_shape_file_extension, - parse_kml_file, - parse_kmz_file, - parse_shapefile, -) +from apps.core.gis.parsers import parse_file_to_geojson from apps.core.models import BackgroundTask, Group, Landscape, User logger = structlog.get_logger(__name__) @@ -56,37 +50,11 @@ class ParseGeoFileView(AuthenticationRequiredMixin, FormView): def post(self, request, **kwargs): file = request.FILES.get("file") - geojson = None - if is_shape_file_extension(file): - try: - geojson = parse_shapefile(file) - except Exception as e: - logger.exception(f"Error when parsing shapefile. File name: {file.name}", error=e) - return JsonResponse( - {"errors": [{"message": json.dumps([{"code": "invalid_shapefile"}])}]}, - status=400, - ) - elif is_kml_file_extension(file): - try: - geojson = parse_kml_file(file) - except Exception as e: - logger.exception(f"Error when parsing KML file. File name: {file.name}", error=e) - return JsonResponse( - {"errors": [{"message": json.dumps([{"code": "invalid_kml_file"}])}]}, - status=400, - ) - elif is_kmz_file_extension(file): - try: - geojson = parse_kmz_file(file) - except Exception as e: - logger.exception(f"Error when parsing KMZ file. File name: {file.name}", error=e) - return JsonResponse( - {"errors": [{"message": json.dumps([{"code": "invalid_kmz_file"}])}]}, - status=400, - ) - else: + try: + geojson = parse_file_to_geojson(file) + except ValidationError as error: return JsonResponse( - {"error": f"File type not supported. File type: {file.content_type}"}, status=400 + {"errors": [{"message": json.dumps([{"code": error.message}])}]}, status=400 ) return JsonResponse({"geojson": geojson}) diff --git a/terraso_backend/apps/graphql/schema/data_entries.py b/terraso_backend/apps/graphql/schema/data_entries.py index 5ba5689d2..783fbf599 100644 --- a/terraso_backend/apps/graphql/schema/data_entries.py +++ b/terraso_backend/apps/graphql/schema/data_entries.py @@ -17,16 +17,20 @@ import graphene import rules import structlog +from django.conf import settings from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError from django.db import transaction from django.db.models import Q from graphene import relay from graphene_django import DjangoObjectType +from apps.core.gis.parsers import parse_file_to_geojson from apps.core.models import Group, Landscape, Membership from apps.graphql.exceptions import GraphQLNotAllowedException, GraphQLNotFoundException from apps.shared_data.models import DataEntry from apps.shared_data.models.data_entries import VALID_TARGET_TYPES +from apps.shared_data.services import data_entry_upload_service from .commons import BaseDeleteMutation, BaseWriteMutation, TerrasoConnection from .constants import MutationTypes @@ -70,6 +74,7 @@ def filter_shared_resources_target_content_type(self, queryset, name, value): class DataEntryNode(DjangoObjectType, SharedResourcesMixin): id = graphene.ID(source="pk", required=True) + geojson = graphene.JSONString() class Meta: model = DataEntry @@ -117,6 +122,15 @@ def resolve_url(self, info): return self.signed_url return self.url + def resolve_geojson(self, info): + if f".{self.resource_type}" not in settings.DATA_ENTRY_GIS_TYPES.keys(): + return None + file = data_entry_upload_service.get_file(self.s3_object_name, "rb") + try: + return parse_file_to_geojson(file) + except ValidationError: + return None + class DataEntryAddMutation(BaseWriteMutation): data_entry = graphene.Field(DataEntryNode) diff --git a/terraso_backend/apps/graphql/schema/schema.graphql b/terraso_backend/apps/graphql/schema/schema.graphql index 6cb5bd914..5402a63f8 100644 --- a/terraso_backend/apps/graphql/schema/schema.graphql +++ b/terraso_backend/apps/graphql/schema/schema.graphql @@ -436,6 +436,7 @@ type VisualizationConfigNode implements Node { createdAt: DateTime! slug: String! title: String! + description: String configuration: JSONString createdBy: UserNode mapboxTilesetId: String @@ -464,6 +465,7 @@ type DataEntryNode implements Node { visualizations(offset: Int, before: String, after: String, first: Int, last: Int, slug: String, slug_Icontains: String, dataEntry_SharedResources_TargetObjectId: UUID, dataEntry_SharedResources_Target_Slug: String, dataEntry_SharedResources_TargetContentType: String): VisualizationConfigNodeConnection! id: ID! sharedResources(offset: Int, before: String, after: String, first: Int, last: Int, source_DataEntry_ResourceType_In: [String]): SharedResourceNodeConnection + geojson: JSONString } """An enumeration.""" @@ -1997,6 +1999,7 @@ type VisualizationConfigAddMutationPayload { input VisualizationConfigAddMutationInput { title: String! + description: String configuration: JSONString dataEntryId: ID! ownerId: ID! diff --git a/terraso_backend/apps/graphql/schema/visualization_config.py b/terraso_backend/apps/graphql/schema/visualization_config.py index bfa82a8d8..6dd0fc0a9 100644 --- a/terraso_backend/apps/graphql/schema/visualization_config.py +++ b/terraso_backend/apps/graphql/schema/visualization_config.py @@ -88,6 +88,7 @@ class Meta: "id", "slug", "title", + "description", "configuration", "created_by", "created_at", @@ -133,6 +134,7 @@ class VisualizationConfigAddMutation(BaseWriteMutation): class Input: title = graphene.String(required=True) + description = graphene.String() configuration = graphene.JSONString() data_entry_id = graphene.ID(required=True) ownerId = graphene.ID(required=True) diff --git a/terraso_backend/apps/shared_data/migrations/0017_visualizationconfig_description.py b/terraso_backend/apps/shared_data/migrations/0017_visualizationconfig_description.py new file mode 100644 index 000000000..9d30301db --- /dev/null +++ b/terraso_backend/apps/shared_data/migrations/0017_visualizationconfig_description.py @@ -0,0 +1,35 @@ +# Copyright © 2023 Technology Matters +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see https://www.gnu.org/licenses/. + +# Generated by Django 4.2.6 on 2023-10-20 22:14 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("shared_data", "0016_remove_dataentry_groups_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="visualizationconfig", + name="description", + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/terraso_backend/apps/shared_data/models/visualization_config.py b/terraso_backend/apps/shared_data/models/visualization_config.py index 1b2d1b354..c49062690 100644 --- a/terraso_backend/apps/shared_data/models/visualization_config.py +++ b/terraso_backend/apps/shared_data/models/visualization_config.py @@ -35,6 +35,7 @@ class VisualizationConfig(SlugModel): ) title = models.CharField(max_length=128, validators=[validate_name]) + description = models.TextField(blank=True, null=True) configuration = models.JSONField(blank=True, null=True) created_by = models.ForeignKey( User, diff --git a/terraso_backend/apps/shared_data/visualization_tileset_tasks.py b/terraso_backend/apps/shared_data/visualization_tileset_tasks.py index a3d0b55a4..b5bba752f 100644 --- a/terraso_backend/apps/shared_data/visualization_tileset_tasks.py +++ b/terraso_backend/apps/shared_data/visualization_tileset_tasks.py @@ -7,6 +7,7 @@ from django.conf import settings from apps.core.gis.mapbox import create_tileset, remove_tileset +from apps.core.gis.parsers import parse_file_to_geojson from apps.shared_data.services import data_entry_upload_service from .models import VisualizationConfig @@ -62,85 +63,105 @@ def get_owner_name(visualization): return visualization.owner.name if visualization.owner else "Unknown" -def create_mapbox_tileset(visualization_id): - logger.info("Creating mapbox tileset", visualization_id=visualization_id) - visualization = VisualizationConfig.objects.get(pk=visualization_id) - data_entry = visualization.data_entry - owner_name = get_owner_name(visualization) +def _get_geojson_from_dataset(data_entry, visualization): + rows = get_rows_from_file(data_entry) - # You cannot update a Mapbox tileset. We have to delete it and create a new one. - remove_mapbox_tileset(visualization.mapbox_tileset_id) + first_row = rows[0] - try: - rows = get_rows_from_file(data_entry) + dataset_config = visualization.configuration["datasetConfig"] + annotate_config = visualization.configuration["annotateConfig"] - first_row = rows[0] + longitude_column = dataset_config["longitude"] + longitude_index = first_row.index(longitude_column) - dataset_config = visualization.configuration["datasetConfig"] - annotate_config = visualization.configuration["annotateConfig"] + latitude_column = dataset_config["latitude"] + latitude_index = first_row.index(latitude_column) - longitude_column = dataset_config["longitude"] - longitude_index = first_row.index(longitude_column) + data_points = annotate_config["dataPoints"] + data_points_indexes = [ + { + "label": data_point.get("label", data_point["column"]), + "index": first_row.index(data_point["column"]), + } + for data_point in data_points + ] - latitude_column = dataset_config["latitude"] - latitude_index = first_row.index(latitude_column) + annotation_title = annotate_config.get("annotationTitle") - data_points = annotate_config["dataPoints"] - data_points_indexes = [ + title_index = ( + first_row.index(annotation_title) + if annotation_title and annotation_title in first_row + else None + ) + + features = [] + for row in rows: + fields = [ { - "label": data_point.get("label", data_point["column"]), - "index": first_row.index(data_point["column"]), + "label": data_point["label"], + "value": row[data_point["index"]], } - for data_point in data_points + for data_point in data_points_indexes ] - annotation_title = annotate_config.get("annotationTitle") - - title_index = ( - first_row.index(annotation_title) - if annotation_title and annotation_title in first_row - else None - ) + properties = { + "title": row[title_index] if title_index else None, + "fields": json.dumps(fields), + } - features = [] - for row in rows: - fields = [ - { - "label": data_point["label"], - "value": row[data_point["index"]], - } - for data_point in data_points_indexes - ] - - properties = { - "title": row[title_index] if title_index else None, - "fields": json.dumps(fields), + try: + longitude = float(row[longitude_index]) + latitude = float(row[latitude_index]) + feature = { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [longitude, latitude], + }, + "properties": properties, } - try: - longitude = float(row[longitude_index]) - latitude = float(row[latitude_index]) - feature = { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [longitude, latitude], - }, - "properties": properties, - } - - features.append(feature) - except ValueError: - continue - - geojson = { - "type": "FeatureCollection", - "features": features, - } + features.append(feature) + except ValueError: + continue + + return { + "type": "FeatureCollection", + "features": features, + } + + +def _get_geojson_from_gis(data_entry): + file = data_entry_upload_service.get_file(data_entry.s3_object_name, "rb") + return parse_file_to_geojson(file) + + +def _get_geojson_from_data_entry(data_entry, visualization): + is_dataset = f".{data_entry.resource_type}" in settings.DATA_ENTRY_SPREADSHEET_TYPES.keys() + is_gis = f".{data_entry.resource_type}" in settings.DATA_ENTRY_GIS_TYPES.keys() + + if is_dataset: + return _get_geojson_from_dataset(data_entry, visualization) + + if is_gis: + return _get_geojson_from_gis(data_entry) + + +def create_mapbox_tileset(visualization_id): + logger.info("Creating mapbox tileset", visualization_id=visualization_id) + visualization = VisualizationConfig.objects.get(pk=visualization_id) + data_entry = visualization.data_entry + owner_name = get_owner_name(visualization) + + # You cannot update a Mapbox tileset. We have to delete it and create a new one. + remove_mapbox_tileset(visualization.mapbox_tileset_id) + + try: + geojson = _get_geojson_from_data_entry(data_entry, visualization) logger.info( "Geojson generated for mapbox tileset", visualization_id=visualization_id, - rows=len(rows), + features=len(geojson["features"]), ) # Include the environment in the title and description when calling the Mapbox API. diff --git a/terraso_backend/tests/core/gis/test_parsers.py b/terraso_backend/tests/core/gis/test_parsers.py index 94ec1c906..1e54e1f83 100644 --- a/terraso_backend/tests/core/gis/test_parsers.py +++ b/terraso_backend/tests/core/gis/test_parsers.py @@ -21,20 +21,105 @@ import geopandas as gpd import pytest -from apps.core.gis.parsers import parse_kml_file, parse_shapefile +from apps.core.gis.parsers import parse_file_to_geojson from apps.core.gis.utils import DEFAULT_CRS +KML_CONTENT = """ + + + + Portland + + -122.681944,45.52,0 + + + + Rio de Janeiro + + -43.196389,-22.908333,0 + + + + Istanbul + + 28.976018,41.01224,0 + + + + Reykjavik + + -21.933333,64.133333,0 + + + + Simple Polygon + + + + -122.681944,45.52,0 + -43.196389,-22.908333,0 + 28.976018,41.01224,0 + -21.933333,64.133333,0 + -122.681944,45.52,0 + + + + + + """ + +KML_GEOJSON = { + "type": "FeatureCollection", + "features": [ + { + "id": "0", + "type": "Feature", + "properties": {"Name": "Portland", "Description": ""}, + "geometry": {"type": "Point", "coordinates": [-122.681944, 45.52, 0.0]}, + }, + { + "id": "1", + "type": "Feature", + "properties": {"Name": "Rio de Janeiro", "Description": ""}, + "geometry": {"type": "Point", "coordinates": [-43.196389, -22.908333, 0.0]}, + }, + { + "id": "2", + "type": "Feature", + "properties": {"Name": "Istanbul", "Description": ""}, + "geometry": {"type": "Point", "coordinates": [28.976018, 41.01224, 0.0]}, + }, + { + "id": "3", + "type": "Feature", + "properties": {"Name": "Reykjavik", "Description": ""}, + "geometry": {"type": "Point", "coordinates": [-21.933333, 64.133333, 0.0]}, + }, + { + "id": "4", + "type": "Feature", + "properties": {"Name": "Simple Polygon", "Description": ""}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-122.681944, 45.52, 0.0], + [-43.196389, -22.908333, 0.0], + [28.976018, 41.01224, 0.0], + [-21.933333, 64.133333, 0.0], + [-122.681944, 45.52, 0.0], + ] + ], + }, + }, + ], +} + @pytest.fixture def shapefile_zip(request): shp, shx, prj = request.param zip_file = tempfile.NamedTemporaryFile(suffix=".zip") - - with zipfile.ZipFile(zip_file.name, "w") as zf: - zf.writestr("test.shp", shp) - zf.writestr("test.shx", shx) - zf.writestr("test.prj", prj) - return zip_file @@ -60,11 +145,12 @@ def test_parse_shapefile(shapefile_zip): for component in ["shp", "shx", "prj"]: zf.write(os.path.join(tmpdir, f"test.{component}"), f"test.{component}") - # Parse the Shapefile - shapefile_json = parse_shapefile(shapefile_zip.name) + with open(shapefile_zip.name, "rb") as file: + shapefile_json = parse_file_to_geojson(file) - # Verify that the parsed Shapefile is equivalent to the original GeoDataFrame - assert shapefile_json == json.loads(gdf.to_json()) + # Verify that the parsed Shapefile is equivalent to the original GeoDataFrame + gdf_json = json.loads(gdf.to_json()) + assert shapefile_json == gdf_json @pytest.fixture @@ -85,103 +171,13 @@ def kml_file(request): @pytest.mark.parametrize( "kml_file", [ - ( - """ - - - - Portland - - -122.681944,45.52,0 - - - - Rio de Janeiro - - -43.196389,-22.908333,0 - - - - Istanbul - - 28.976018,41.01224,0 - - - - Reykjavik - - -21.933333,64.133333,0 - - - - Simple Polygon - - - - -122.681944,45.52,0 - -43.196389,-22.908333,0 - 28.976018,41.01224,0 - -21.933333,64.133333,0 - -122.681944,45.52,0 - - - - - - """, - "kml", - ), + (KML_CONTENT, "kml"), ], indirect=True, ) def test_parse_kml_file(kml_file): - # Call the parse_kml_file function with the file path - kml_json = parse_kml_file(kml_file) + with open(kml_file, "rb") as file: + kml_json = parse_file_to_geojson(file) # Assert that the output of the parse_kml_file function is as expected - assert kml_json == { - "type": "FeatureCollection", - "features": [ - { - "id": "0", - "type": "Feature", - "properties": {"Name": "Portland", "Description": ""}, - "geometry": {"type": "Point", "coordinates": [-122.681944, 45.52, 0.0]}, - }, - { - "id": "1", - "type": "Feature", - "properties": {"Name": "Rio de Janeiro", "Description": ""}, - "geometry": {"type": "Point", "coordinates": [-43.196389, -22.908333, 0.0]}, - }, - { - "id": "2", - "type": "Feature", - "properties": {"Name": "Istanbul", "Description": ""}, - "geometry": {"type": "Point", "coordinates": [28.976018, 41.01224, 0.0]}, - }, - { - "id": "3", - "type": "Feature", - "properties": {"Name": "Reykjavik", "Description": ""}, - "geometry": {"type": "Point", "coordinates": [-21.933333, 64.133333, 0.0]}, - }, - { - "id": "4", - "type": "Feature", - "properties": {"Name": "Simple Polygon", "Description": ""}, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [-122.681944, 45.52, 0.0], - [-43.196389, -22.908333, 0.0], - [28.976018, 41.01224, 0.0], - [-21.933333, 64.133333, 0.0], - [-122.681944, 45.52, 0.0], - ] - ], - }, - }, - ], - } + assert kml_json == KML_GEOJSON diff --git a/terraso_backend/tests/graphql/conftest.py b/terraso_backend/tests/graphql/conftest.py index e6db1caf2..02478a8a5 100644 --- a/terraso_backend/tests/graphql/conftest.py +++ b/terraso_backend/tests/graphql/conftest.py @@ -318,6 +318,36 @@ def data_entries(group_data_entries, landscape_data_entries): return group_data_entries + landscape_data_entries +@pytest.fixture +def data_entry_kml(users, groups): + creator = users[0] + creator_group = groups[0] + creator_group.members.add(creator) + return mixer.blend( + DataEntry, + created_by=creator, + size=100, + groups=creator_group, + entry_type=DataEntry.ENTRY_TYPE_FILE, + resource_type="kml", + ) + + +@pytest.fixture +def data_entry_shapefile(users, groups): + creator = users[0] + creator_group = groups[0] + creator_group.members.add(creator) + return mixer.blend( + DataEntry, + created_by=creator, + size=100, + groups=creator_group, + entry_type=DataEntry.ENTRY_TYPE_FILE, + resource_type="zip", + ) + + @pytest.fixture def visualization_config_current_user(users, data_entry_current_user_file, groups): creator = users[0] diff --git a/terraso_backend/tests/graphql/test_shared_data.py b/terraso_backend/tests/graphql/test_shared_data.py index 75e5b42e8..73223ec5b 100644 --- a/terraso_backend/tests/graphql/test_shared_data.py +++ b/terraso_backend/tests/graphql/test_shared_data.py @@ -13,8 +13,19 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see https://www.gnu.org/licenses/. +import json +import os +import tempfile +import zipfile +from unittest import mock + +import geopandas as gpd import pytest +from apps.core.gis.utils import DEFAULT_CRS + +from ..core.gis.test_parsers import KML_CONTENT, KML_GEOJSON + pytestmark = pytest.mark.django_db @@ -289,3 +300,112 @@ def test_data_entries_from_parent_query_by_resource_field(client_query, data_ent for data_entry in data_entries: assert data_entry.name in entries_result + + +@pytest.fixture +def kml_file(request): + kml_contents, file_extension = request.param + # Create a temporary file + with tempfile.NamedTemporaryFile(mode="w", suffix=f".{file_extension}", delete=False) as f: + # Write the KML content to the file + f.write(kml_contents) + + # Return the file path + yield f.name + + # Clean up: delete the temporary file + os.unlink(f.name) + + +@pytest.mark.parametrize( + "kml_file", + [ + ( + KML_CONTENT, + "kml", + ), + ], + indirect=True, +) +@mock.patch("apps.shared_data.services.data_entry_upload_service.get_file") +def test_data_entry_kml_to_geojson(get_file_mock, client_query, data_entry_kml, kml_file): + with open(kml_file, "rb") as file: + get_file_mock.return_value = file + response = client_query( + """ + {dataEntry(id: "%s") { + id + name + geojson + }} + """ + % data_entry_kml.id + ) + json_response = response.json() + data_entry_result = json_response["data"]["dataEntry"] + + assert data_entry_result["id"] == str(data_entry_kml.id) + assert data_entry_result["name"] == data_entry_kml.name + assert data_entry_result["geojson"] == json.dumps(KML_GEOJSON) + + +@mock.patch("apps.shared_data.services.data_entry_upload_service.get_file") +def test_data_entry_shapefil_to_geojson(get_file_mock, client_query, data_entry_shapefile): + gdf = gpd.GeoDataFrame({"geometry": gpd.points_from_xy([0], [0])}, crs=DEFAULT_CRS) + with tempfile.TemporaryDirectory() as tmpdir: + shapefile_zip = tempfile.NamedTemporaryFile(suffix=".zip") + shapefile_path = os.path.join(tmpdir, "test.shp") + gdf.to_file(shapefile_path) + + with zipfile.ZipFile(shapefile_zip.name, "w") as zf: + for component in ["shp", "shx", "prj"]: + zf.write(os.path.join(tmpdir, f"test.{component}"), f"test.{component}") + + with open(shapefile_zip.name, "rb") as file: + get_file_mock.return_value = file + response = client_query( + """ + {dataEntry(id: "%s") { + id + name + geojson + }} + """ + % data_entry_shapefile.id + ) + json_response = response.json() + data_entry_result = json_response["data"]["dataEntry"] + + assert data_entry_result["id"] == str(data_entry_shapefile.id) + assert data_entry_result["name"] == data_entry_shapefile.name + assert json.loads(data_entry_result["geojson"]) == { + "type": "FeatureCollection", + "features": [ + { + "id": "0", + "type": "Feature", + "properties": {}, + "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, + } + ], + "crs": {"type": "name", "properties": {"name": "urn:ogc:def:crs:OGC::CRS84"}}, + } + + +@mock.patch("apps.shared_data.services.data_entry_upload_service.get_file") +def test_data_entry_avoid_fetching_file_for_not_gis_file(get_file_mock, client_query, data_entries): + response = client_query( + """ + {dataEntry(id: "%s") { + id + name + geojson + }} + """ + % data_entries[0].id + ) + json_response = response.json() + data_entry_result = json_response["data"]["dataEntry"] + + get_file_mock.assert_not_called() + assert data_entry_result["geojson"] is None diff --git a/terraso_backend/tests/graphql/test_visualization_config.py b/terraso_backend/tests/graphql/test_visualization_config.py index 8612223e3..d03cbc317 100644 --- a/terraso_backend/tests/graphql/test_visualization_config.py +++ b/terraso_backend/tests/graphql/test_visualization_config.py @@ -170,6 +170,8 @@ def test_visualization_configs_returns_only_for_users_groups( edges { node { id + title + description } } }} @@ -181,3 +183,5 @@ def test_visualization_configs_returns_only_for_users_groups( assert len(entries_result) == 1 assert entries_result[0] == str(visualization_config_current_user.id) + assert edges[0]["node"]["title"] == visualization_config_current_user.title + assert edges[0]["node"]["description"] == visualization_config_current_user.description diff --git a/terraso_backend/tests/shared_data/conftest.py b/terraso_backend/tests/shared_data/conftest.py index 407daf294..910409234 100644 --- a/terraso_backend/tests/shared_data/conftest.py +++ b/terraso_backend/tests/shared_data/conftest.py @@ -85,3 +85,19 @@ def visualization_config_b(user_b, data_entry): data_entry=data_entry, created_by=user_b, ) + + +@pytest.fixture +def visualization_config_kml(user): + return mixer.blend( + VisualizationConfig, + size=1, + data_entry=mixer.blend( + DataEntry, + size=1, + url=f"{settings.DATA_ENTRY_FILE_BASE_URL}/{user.id}/test_data.kml", + created_by=user, + resource_type="kml", + ), + created_by=user, + ) diff --git a/terraso_backend/tests/shared_data/test_visualization_tileset_tasks.py b/terraso_backend/tests/shared_data/test_visualization_tileset_tasks.py index d5fca3c25..299350be4 100644 --- a/terraso_backend/tests/shared_data/test_visualization_tileset_tasks.py +++ b/terraso_backend/tests/shared_data/test_visualization_tileset_tasks.py @@ -14,17 +14,23 @@ # along with this program. If not, see https://www.gnu.org/licenses/. import io +import json +import os +import tempfile from unittest.mock import patch import pytest import requests +from apps.core.gis.mapbox import get_line_delimited_geojson from apps.shared_data.models import VisualizationConfig from apps.shared_data.visualization_tileset_tasks import ( create_mapbox_tileset, remove_mapbox_tileset, ) +from ..core.gis.test_parsers import KML_CONTENT, KML_GEOJSON + pytestmark = pytest.mark.django_db @@ -37,7 +43,9 @@ def create_mock_response(response): @patch("apps.shared_data.visualization_tileset_tasks.data_entry_upload_service.get_file") @patch("apps.core.gis.mapbox.requests.post") -def test_create_mapbox_tileset_success(mock_request_post, mock_get_file, visualization_config): +def test_create_mapbox_tileset_dataset_success( + mock_request_post, mock_get_file, visualization_config +): visualization_config.configuration = { "datasetConfig": { "longitude": "lng", @@ -53,7 +61,9 @@ def test_create_mapbox_tileset_success(mock_request_post, mock_get_file, visuali }, } visualization_config.save() - mock_get_file.return_value = io.StringIO("lat,lng,col1\nval1,val2,val3") + mock_get_file.return_value = io.StringIO( + "lat,lng,col1\n-78.48306234911033,-0.1805502450716432,val3" + ) mock_responses = [ {"status_code": 200, "json_data": {"id": "tileset-id-1"}}, {"status_code": 200, "json_data": {}}, @@ -65,6 +75,63 @@ def test_create_mapbox_tileset_success(mock_request_post, mock_get_file, visuali assert updated_visualization_config.mapbox_tileset_id is not None assert mock_request_post.call_count == 3 + assert json.loads(mock_request_post.call_args_list[0][1]["files"][0][1][1]) == { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [-0.1805502450716432, -78.48306234911033]}, + "properties": { + "title": None, + "fields": '[{"label": "label", "value": "val3"}]', + }, + } + + assert ( + mock_request_post.call_args_list[1][1]["json"]["name"] + == f"development - {visualization_config.title}"[:64] + ) + + +@pytest.fixture +def kml_file(): + kml_contents = KML_CONTENT + file_extension = "kml" + with tempfile.NamedTemporaryFile(mode="w", suffix=f".{file_extension}", delete=False) as f: + f.write(kml_contents) + + yield f.name + + os.unlink(f.name) + + +@patch("apps.shared_data.services.data_entry_upload_service.get_file") +@patch("apps.core.gis.mapbox.requests.post") +def test_create_mapbox_tileset_gis_dataentry_success( + mock_request_post, mock_get_file, visualization_config_kml, kml_file +): + with open(kml_file, "rb") as file: + mock_get_file.return_value = file + mock_responses = [ + {"status_code": 200, "json_data": {"id": "tileset-id-1"}}, + {"status_code": 200, "json_data": {}}, + {"status_code": 200, "json_data": {}}, + ] + mock_request_post.side_effect = [ + create_mock_response(response) for response in mock_responses + ] + create_mapbox_tileset(visualization_config_kml.id) + + updated_visualization_config = VisualizationConfig.objects.get(id=visualization_config_kml.id) + assert updated_visualization_config.mapbox_tileset_id is not None + assert mock_request_post.call_count == 3 + + assert mock_request_post.call_args_list[0][1]["files"][0][1][1] == get_line_delimited_geojson( + KML_GEOJSON + ) + + assert ( + mock_request_post.call_args_list[1][1]["json"]["name"] + == f"development - {visualization_config_kml.title}"[:64] + ) + @patch("apps.shared_data.visualization_tileset_tasks.data_entry_upload_service.get_file") @patch("apps.core.gis.mapbox.requests.post")