Skip to content

Commit

Permalink
fix: Read KML layers, handle kml with no layers
Browse files Browse the repository at this point in the history
  • Loading branch information
@josebui
josebui committed Nov 30, 2023
1 parent 41d6577 commit 4768ca2
Show file tree
Hide file tree
Showing 7 changed files with 1,337 additions and 1,401 deletions.
34 changes: 17 additions & 17 deletions terraso_backend/apps/core/gis/parsers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see https://www.gnu.org/licenses/.

import io
import json
import os
import uuid
import zipfile

import fiona
import geopandas as gpd
import pandas as pd
import structlog
Expand All @@ -30,9 +30,8 @@

logger = structlog.get_logger(__name__)

supported_drivers["KML"] = "rw"
supported_drivers["GPX"] = "rw"
supported_drivers["LIBKML"] = "rw"
supported_drivers["GPX"] = "rw"

IGNORE_KML_PROPS = [
"tessellate",
Expand Down Expand Up @@ -74,22 +73,23 @@ def is_gpx_file_extension(file):
return file.name.endswith(".gpx")


def parse_kml_file(file_buffer):
kml_drivers = ["LIBKML", "KML"]
# gdf_kml = gpd.read_file(file, driver="KML")
# Try all KML drivers and use the one that gathers the most data
gdf = None
for driver in kml_drivers:
try:
file_buffer.seek(0)
def _get_kml_gdf(file_buffer):
layers = fiona.listlayers(file_buffer)

with io.BytesIO(file_buffer.read()) as memfile:
new_gdf = gpd.read_file(memfile, driver=driver)
if gdf is None or len(new_gdf) > len(gdf):
gdf = new_gdf
if len(layers) == 1:
file_buffer.seek(0)
return gpd.read_file(file_buffer, driver="LIBKML")

except Exception as e:
logger.exception("Error parsing kml file", error=e)
combined_gdf = gpd.GeoDataFrame()
for layer in layers:
file_buffer.seek(0)
gdf = gpd.read_file(file_buffer, driver="LIBKML", layer=layer)
combined_gdf = pd.concat([combined_gdf, gdf], ignore_index=True)
return combined_gdf


def parse_kml_file(file_buffer):
gdf = _get_kml_gdf(file_buffer)

def row_to_dict(row):
row_dict = row.to_dict()
Expand Down
110 changes: 6 additions & 104 deletions terraso_backend/tests/core/gis/test_parsers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,49 +25,11 @@
from apps.core.gis.parsers import parse_file_to_geojson
from apps.core.gis.utils import DEFAULT_CRS

KML_CONTENT = """<?xml version="1.0" encoding="utf-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
<Placemark>
<name>Portland</name>
<Point>
<coordinates>-122.681944,45.52,0</coordinates>
</Point>
</Placemark>
<Placemark>
<name>Rio de Janeiro</name>
<Point>
<coordinates>-43.196389,-22.908333,0</coordinates>
</Point>
</Placemark>
<Placemark>
<name>Istanbul</name>
<Point>
<coordinates>28.976018,41.01224,0</coordinates>
</Point>
</Placemark>
<Placemark>
<name>Reykjavik</name>
<Point>
<coordinates>-21.933333,64.133333,0</coordinates>
</Point>
</Placemark>
<Placemark>
<name>Simple Polygon</name>
<Polygon>
<outerBoundaryIs>
<LinearRing>
<coordinates>-122.681944,45.52,0
-43.196389,-22.908333,0
28.976018,41.01224,0
-21.933333,64.133333,0
-122.681944,45.52,0</coordinates>
</LinearRing>
</outerBoundaryIs>
</Polygon>
</Placemark>
</Document>
</kml>"""
KML_TEST_FILES = [
("resources/gis/kml_sample_1.kml", "resources/gis/kml_sample_1_geojson.json"),
("resources/gis/kml_sample_2.kml", "resources/gis/kml_sample_2_geojson.json"),
("resources/gis/kml_sample_3.kml", "resources/gis/kml_sample_3_geojson.json"),
]

GPX_CONTENT = """<?xml version="1.0" standalone="yes"?>
<gpx xmlns="http://www.topografix.com/GPX/1/1" version="1.1"
Expand Down Expand Up @@ -99,48 +61,6 @@
</wpt>
</gpx>"""

KML_GEOJSON = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {"Name": "Portland"},
"geometry": {"type": "Point", "coordinates": [-122.681944, 45.52, 0.0]},
},
{
"type": "Feature",
"properties": {"Name": "Rio de Janeiro"},
"geometry": {"type": "Point", "coordinates": [-43.196389, -22.908333, 0.0]},
},
{
"type": "Feature",
"properties": {"Name": "Istanbul"},
"geometry": {"type": "Point", "coordinates": [28.976018, 41.01224, 0.0]},
},
{
"type": "Feature",
"properties": {"Name": "Reykjavik"},
"geometry": {"type": "Point", "coordinates": [-21.933333, 64.133333, 0.0]},
},
{
"type": "Feature",
"properties": {"Name": "Simple Polygon"},
"geometry": {
"type": "Polygon",
"coordinates": [
[
[-122.681944, 45.52, 0.0],
[-43.196389, -22.908333, 0.0],
[28.976018, 41.01224, 0.0],
[-21.933333, 64.133333, 0.0],
[-122.681944, 45.52, 0.0],
]
],
},
},
],
}

GPX_GEOJSON = {
"type": "FeatureCollection",
"features": [
Expand Down Expand Up @@ -305,27 +225,9 @@ def test_parse_shapefile(shapefile_zip):
assert shapefile_json == gdf_json


# @pytest.fixture
# def kml_file(request):
# kml_contents, file_extension = request.param
# # Create a temporary file
# with tempfile.NamedTemporaryFile(mode="w", suffix=f".{file_extension}", delete=False) as f:
# # Write the KML content to the file
# f.write(kml_contents)

# # Return the file path
# yield f.name

# # Clean up: delete the temporary file
# os.unlink(f.name)


@pytest.mark.parametrize(
"kml_file_path_expected",
[
("resources/gis/kml_sample_1.kml", "resources/gis/kml_sample_1_geojson.json"),
# ("resources/gis/kml_sample_2.kml", "resources/gis/kml_sample_2_geojson.json"),
],
KML_TEST_FILES,
)
def test_parse_kml_file(kml_file_path_expected):
kml_file_path = kml_file_path_expected[0]
Expand Down
40 changes: 13 additions & 27 deletions terraso_backend/tests/graphql/test_shared_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,15 @@
import os
import tempfile
import zipfile
from importlib import resources
from unittest import mock

import geopandas as gpd
import pytest

from apps.core.gis.utils import DEFAULT_CRS

from ..core.gis.test_parsers import KML_CONTENT, KML_GEOJSON
from ..core.gis.test_parsers import KML_TEST_FILES

pytestmark = pytest.mark.django_db

Expand Down Expand Up @@ -302,34 +303,19 @@ def test_data_entries_from_parent_query_by_resource_field(client_query, data_ent
assert data_entry.name in entries_result


@pytest.fixture
def kml_file(request):
kml_contents, file_extension = request.param
# Create a temporary file
with tempfile.NamedTemporaryFile(mode="w", suffix=f".{file_extension}", delete=False) as f:
# Write the KML content to the file
f.write(kml_contents)

# Return the file path
yield f.name

# Clean up: delete the temporary file
os.unlink(f.name)


@pytest.mark.parametrize(
"kml_file",
[
(
KML_CONTENT,
"kml",
),
],
indirect=True,
"kml_file_path_expected",
KML_TEST_FILES,
)
@mock.patch("apps.shared_data.services.data_entry_upload_service.get_file")
def test_data_entry_kml_to_geojson(get_file_mock, client_query, data_entry_kml, kml_file):
with open(kml_file, "rb") as file:
def test_data_entry_kml_to_geojson(
get_file_mock, client_query, data_entry_kml, kml_file_path_expected
):
expected_file_path = kml_file_path_expected[1]
with open(resources.files("tests").joinpath(expected_file_path), "rb") as file:
expected_json = json.load(file)
kml_file_path = kml_file_path_expected[0]
with open(resources.files("tests").joinpath(kml_file_path), "rb") as file:
get_file_mock.return_value = file
response = client_query(
"""
Expand All @@ -346,7 +332,7 @@ def test_data_entry_kml_to_geojson(get_file_mock, client_query, data_entry_kml,

assert data_entry_result["id"] == str(data_entry_kml.id)
assert data_entry_result["name"] == data_entry_kml.name
assert json.loads(data_entry_result["geojson"])["features"] == KML_GEOJSON["features"]
assert json.loads(data_entry_result["geojson"])["features"] == expected_json["features"]


@mock.patch("apps.shared_data.services.data_entry_upload_service.get_file")
Expand Down
Loading

0 comments on commit 4768ca2

Please sign in to comment.