Skip to content

Commit

Permalink
feat: Visualization (data map) GIS formats support (#917)
Browse files Browse the repository at this point in the history
  • Loading branch information
josebui authored Nov 8, 2023
1 parent 543edfe commit 2162f0e
Show file tree
Hide file tree
Showing 15 changed files with 527 additions and 209 deletions.
8 changes: 6 additions & 2 deletions terraso_backend/apps/core/gis/mapbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,15 @@ def get_publish_status(id):
return len(response_json) > 0


def get_line_delimited_geojson(geojson):
return "\n".join([json.dumps(feature) for feature in geojson["features"]])


def _post_tileset_source(geojson, id):
line_delimited_geojson = "\n".join([json.dumps(feature) for feature in geojson["features"]])
line_delimited_geojson = get_line_delimited_geojson(geojson)

url = f"{API_URL}/tilesets/v1/sources/{USERNAME}/{id}?access_token={TOKEN}"
multipart_data = [("file", ("test.ndjson", line_delimited_geojson, "text/plain"))]
multipart_data = [("file", ("input.ndjson", line_delimited_geojson, "text/plain"))]

response = requests.post(url, files=multipart_data)
response_json = response.json()
Expand Down
37 changes: 37 additions & 0 deletions terraso_backend/apps/core/gis/parsers.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,21 @@
import zipfile

import geopandas as gpd
import structlog
from django.core.exceptions import ValidationError
from fiona.drvsupport import supported_drivers

from apps.core.gis.utils import DEFAULT_CRS

logger = structlog.get_logger(__name__)

supported_drivers["KML"] = "rw"


def is_geojson_file_extension(file):
return file.name.endswith((".geojson", ".json"))


def is_shape_file_extension(file):
return file.name.endswith(".zip")

Expand Down Expand Up @@ -96,3 +104,32 @@ def parse_shapefile(file):
os.rmdir(tmp_folder)

return json.loads(gdf_transformed.to_json())


def parse_file_to_geojson(file):
if is_shape_file_extension(file):
try:
return parse_shapefile(file)
except Exception as e:
logger.error("Error parsing shapefile", error=e)
raise ValidationError("invalid_shapefile")
elif is_kml_file_extension(file):
try:
return parse_kml_file(file)
except Exception as e:
logger.error("Error parsing kml file", error=e)
raise ValidationError("invalid_kml_file")
elif is_kmz_file_extension(file):
try:
return parse_kmz_file(file)
except Exception as e:
logger.error("Error parsing kmz file", error=e)
raise ValidationError("invalid_kmz_file")
elif is_geojson_file_extension(file):
try:
return json.load(file)
except Exception as e:
logger.error("Error parsing geojson file", error=e)
raise ValidationError("invalid_geojson_file")
else:
raise ValidationError("invalid_file_type")
44 changes: 6 additions & 38 deletions terraso_backend/apps/core/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.sessions.models import Session
from django.core import management
from django.core.exceptions import ValidationError
from django.db import DatabaseError, transaction
from django.db.transaction import get_connection
from django.http import (
Expand All @@ -37,14 +38,7 @@
from django.views.generic.edit import FormView

from apps.auth.mixins import AuthenticationRequiredMixin
from apps.core.gis.parsers import (
is_kml_file_extension,
is_kmz_file_extension,
is_shape_file_extension,
parse_kml_file,
parse_kmz_file,
parse_shapefile,
)
from apps.core.gis.parsers import parse_file_to_geojson
from apps.core.models import BackgroundTask, Group, Landscape, User

logger = structlog.get_logger(__name__)
Expand All @@ -56,37 +50,11 @@ class ParseGeoFileView(AuthenticationRequiredMixin, FormView):
def post(self, request, **kwargs):
file = request.FILES.get("file")

geojson = None
if is_shape_file_extension(file):
try:
geojson = parse_shapefile(file)
except Exception as e:
logger.exception(f"Error when parsing shapefile. File name: {file.name}", error=e)
return JsonResponse(
{"errors": [{"message": json.dumps([{"code": "invalid_shapefile"}])}]},
status=400,
)
elif is_kml_file_extension(file):
try:
geojson = parse_kml_file(file)
except Exception as e:
logger.exception(f"Error when parsing KML file. File name: {file.name}", error=e)
return JsonResponse(
{"errors": [{"message": json.dumps([{"code": "invalid_kml_file"}])}]},
status=400,
)
elif is_kmz_file_extension(file):
try:
geojson = parse_kmz_file(file)
except Exception as e:
logger.exception(f"Error when parsing KMZ file. File name: {file.name}", error=e)
return JsonResponse(
{"errors": [{"message": json.dumps([{"code": "invalid_kmz_file"}])}]},
status=400,
)
else:
try:
geojson = parse_file_to_geojson(file)
except ValidationError as error:
return JsonResponse(
{"error": f"File type not supported. File type: {file.content_type}"}, status=400
{"errors": [{"message": json.dumps([{"code": error.message}])}]}, status=400
)

return JsonResponse({"geojson": geojson})
Expand Down
14 changes: 14 additions & 0 deletions terraso_backend/apps/graphql/schema/data_entries.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,20 @@
import graphene
import rules
import structlog
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.db import transaction
from django.db.models import Q
from graphene import relay
from graphene_django import DjangoObjectType

from apps.core.gis.parsers import parse_file_to_geojson
from apps.core.models import Group, Landscape, Membership
from apps.graphql.exceptions import GraphQLNotAllowedException, GraphQLNotFoundException
from apps.shared_data.models import DataEntry
from apps.shared_data.models.data_entries import VALID_TARGET_TYPES
from apps.shared_data.services import data_entry_upload_service

from .commons import BaseDeleteMutation, BaseWriteMutation, TerrasoConnection
from .constants import MutationTypes
Expand Down Expand Up @@ -70,6 +74,7 @@ def filter_shared_resources_target_content_type(self, queryset, name, value):

class DataEntryNode(DjangoObjectType, SharedResourcesMixin):
id = graphene.ID(source="pk", required=True)
geojson = graphene.JSONString()

class Meta:
model = DataEntry
Expand Down Expand Up @@ -117,6 +122,15 @@ def resolve_url(self, info):
return self.signed_url
return self.url

def resolve_geojson(self, info):
if f".{self.resource_type}" not in settings.DATA_ENTRY_GIS_TYPES.keys():
return None
file = data_entry_upload_service.get_file(self.s3_object_name, "rb")
try:
return parse_file_to_geojson(file)
except ValidationError:
return None


class DataEntryAddMutation(BaseWriteMutation):
data_entry = graphene.Field(DataEntryNode)
Expand Down
3 changes: 3 additions & 0 deletions terraso_backend/apps/graphql/schema/schema.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,7 @@ type VisualizationConfigNode implements Node {
createdAt: DateTime!
slug: String!
title: String!
description: String
configuration: JSONString
createdBy: UserNode
mapboxTilesetId: String
Expand Down Expand Up @@ -464,6 +465,7 @@ type DataEntryNode implements Node {
visualizations(offset: Int, before: String, after: String, first: Int, last: Int, slug: String, slug_Icontains: String, dataEntry_SharedResources_TargetObjectId: UUID, dataEntry_SharedResources_Target_Slug: String, dataEntry_SharedResources_TargetContentType: String): VisualizationConfigNodeConnection!
id: ID!
sharedResources(offset: Int, before: String, after: String, first: Int, last: Int, source_DataEntry_ResourceType_In: [String]): SharedResourceNodeConnection
geojson: JSONString
}

"""An enumeration."""
Expand Down Expand Up @@ -1997,6 +1999,7 @@ type VisualizationConfigAddMutationPayload {

input VisualizationConfigAddMutationInput {
title: String!
description: String
configuration: JSONString
dataEntryId: ID!
ownerId: ID!
Expand Down
2 changes: 2 additions & 0 deletions terraso_backend/apps/graphql/schema/visualization_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ class Meta:
"id",
"slug",
"title",
"description",
"configuration",
"created_by",
"created_at",
Expand Down Expand Up @@ -133,6 +134,7 @@ class VisualizationConfigAddMutation(BaseWriteMutation):

class Input:
title = graphene.String(required=True)
description = graphene.String()
configuration = graphene.JSONString()
data_entry_id = graphene.ID(required=True)
ownerId = graphene.ID(required=True)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# Copyright © 2023 Technology Matters
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see https://www.gnu.org/licenses/.

# Generated by Django 4.2.6 on 2023-10-20 22:14

import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("shared_data", "0016_remove_dataentry_groups_and_more"),
]

operations = [
migrations.AddField(
model_name="visualizationconfig",
name="description",
field=models.TextField(blank=True, null=True),
),
]
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ class VisualizationConfig(SlugModel):
)

title = models.CharField(max_length=128, validators=[validate_name])
description = models.TextField(blank=True, null=True)
configuration = models.JSONField(blank=True, null=True)
created_by = models.ForeignKey(
User,
Expand Down
Loading

0 comments on commit 2162f0e

Please sign in to comment.