From ca7aa514699323afff3227aa4487debda511e04a Mon Sep 17 00:00:00 2001 From: Jose Buitron Date: Tue, 12 Dec 2023 10:57:36 -0500 Subject: [PATCH 1/2] fix: Delete all folders created by shapefile processing --- terraso_backend/apps/core/gis/parsers.py | 9 ++-- .../tests/core/gis/test_parsers.py | 49 ++++++------------ .../resources/gis/shapefile_sample_1.zip | Bin 0 -> 2121 bytes .../gis/shapefile_sample_1_geojson.json | 25 +++++++++ .../resources/gis/shapefile_sample_2.zip | Bin 0 -> 958 bytes .../gis/shapefile_sample_2_geojson.json | 25 +++++++++ 6 files changed, 70 insertions(+), 38 deletions(-) create mode 100644 terraso_backend/tests/resources/gis/shapefile_sample_1.zip create mode 100644 terraso_backend/tests/resources/gis/shapefile_sample_1_geojson.json create mode 100644 terraso_backend/tests/resources/gis/shapefile_sample_2.zip create mode 100644 terraso_backend/tests/resources/gis/shapefile_sample_2_geojson.json diff --git a/terraso_backend/apps/core/gis/parsers.py b/terraso_backend/apps/core/gis/parsers.py index 94a63986f..16b771dd8 100644 --- a/terraso_backend/apps/core/gis/parsers.py +++ b/terraso_backend/apps/core/gis/parsers.py @@ -157,10 +157,11 @@ def parse_shapefile(file): gdf_transformed = gdf.to_crs(crs=DEFAULT_CRS) # Delete extracted files - os.remove(os.path.join(tmp_folder, shp_filenames[0])) - os.remove(os.path.join(tmp_folder, shx_filenames[0])) - os.remove(os.path.join(tmp_folder, prj_filenames[0])) - os.rmdir(tmp_folder) + for root, dirs, files in os.walk(tmp_folder, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) return json.loads(gdf_transformed.to_json()) diff --git a/terraso_backend/tests/core/gis/test_parsers.py b/terraso_backend/tests/core/gis/test_parsers.py index 26b481ff5..279a90902 100644 --- a/terraso_backend/tests/core/gis/test_parsers.py +++ b/terraso_backend/tests/core/gis/test_parsers.py @@ -16,20 +16,21 @@ import json import os import tempfile -import zipfile from importlib import resources -import geopandas as gpd import pytest from apps.core.gis.parsers import parse_file_to_geojson -from apps.core.gis.utils import DEFAULT_CRS KML_TEST_FILES = [ ("resources/gis/kml_sample_1.kml", "resources/gis/kml_sample_1_geojson.json"), ("resources/gis/kml_sample_2.kml", "resources/gis/kml_sample_2_geojson.json"), ("resources/gis/kml_sample_3.kml", "resources/gis/kml_sample_3_geojson.json"), ] +SHAPEFILE_TEST_FILES = [ + ("resources/gis/shapefile_sample_1.zip", "resources/gis/shapefile_sample_1_geojson.json"), + ("resources/gis/shapefile_sample_2.zip", "resources/gis/shapefile_sample_2_geojson.json"), +] GPX_CONTENT = """ ", b"", b""), - (b"", b"", b""), - ], - indirect=True, + "file_path_expected", + SHAPEFILE_TEST_FILES, ) -def test_parse_shapefile(shapefile_zip): - # Create a GeoDataFrame with a single point - gdf = gpd.GeoDataFrame({"geometry": gpd.points_from_xy([0], [0])}, crs=DEFAULT_CRS) - - # Convert the GeoDataFrame to a Shapefile - with tempfile.TemporaryDirectory() as tmpdir: - shapefile_path = os.path.join(tmpdir, "test.shp") - gdf.to_file(shapefile_path) - - # Zip the Shapefile components - with zipfile.ZipFile(shapefile_zip.name, "w") as zf: - for component in ["shp", "shx", "prj"]: - zf.write(os.path.join(tmpdir, f"test.{component}"), f"test.{component}") +def test_parse_shapefile(file_path_expected): + file_path = file_path_expected[0] + with open(resources.files("tests").joinpath(file_path), "rb") as file: + shapefile_json = parse_file_to_geojson(file) - with open(shapefile_zip.name, "rb") as file: - shapefile_json = parse_file_to_geojson(file) + expected_file_path = file_path_expected[1] + with open(resources.files("tests").joinpath(expected_file_path), "rb") as file: + expected_json = json.load(file) - # Verify that the parsed Shapefile is equivalent to the original GeoDataFrame - gdf_json = json.loads(gdf.to_json()) - assert shapefile_json == gdf_json + print(f"shapefile_json: {shapefile_json}") + assert json.dumps(shapefile_json) == json.dumps(expected_json) @pytest.mark.parametrize( diff --git a/terraso_backend/tests/resources/gis/shapefile_sample_1.zip b/terraso_backend/tests/resources/gis/shapefile_sample_1.zip new file mode 100644 index 0000000000000000000000000000000000000000..60e94353ad8c1a074f1ee86b3e08a3d30abd0689 GIT binary patch literal 2121 zcmWIWW@Zs#-~htLfSzy$C{PB{oD2#K0sfwTA$loEX`vy!4D3l|6{#Qvr9is0f}4Sn zqjAFXsKx zVrAvzVB^O(BIY(I_{WDYsQbR?%9TBz&OG5!m2Hlg8(@E4V8P6lFC79GG|X#UxU+NR z#)(TOu3WfuDd&VlDQRx7e;ywU%&G*s802c~{=pe3=w2JlkycF~S^< zUK~*5@P9yKF&vI=1C{~@-Qma?6V>5|nP3h_&VityqsZYd%vc(GYi$V8>|z*8XFtn&B!FefIIO)ybJOO2*6SqnkMwb2vH0o zRgpBok|w&9=!pZN`!c5P3UuA*5s1*83ye3IgTZzKqZ3^>dTb$d??cxOjJp7DRyL3d NRv^>`%2zXicmUPYv^M|% literal 0 HcmV?d00001 diff --git a/terraso_backend/tests/resources/gis/shapefile_sample_2_geojson.json b/terraso_backend/tests/resources/gis/shapefile_sample_2_geojson.json new file mode 100644 index 000000000..ad70b993e --- /dev/null +++ b/terraso_backend/tests/resources/gis/shapefile_sample_2_geojson.json @@ -0,0 +1,25 @@ +{ + "type":"FeatureCollection", + "features":[ + { + "id":"0", + "type":"Feature", + "properties":{ + + }, + "geometry":{ + "type":"Point", + "coordinates":[ + -78.48247445411837, + -0.16708426936664011 + ] + } + } + ], + "crs":{ + "type":"name", + "properties":{ + "name":"urn:ogc:def:crs:OGC::CRS84" + } + } +} \ No newline at end of file From 7bba3965bf0ec0ba7c8c9ca3bafc1fe6cbd36396 Mon Sep 17 00:00:00 2001 From: Jose Buitron Date: Thu, 14 Dec 2023 12:58:56 -0500 Subject: [PATCH 2/2] fix: Use shutil to delete temp folder --- terraso_backend/apps/core/gis/parsers.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/terraso_backend/apps/core/gis/parsers.py b/terraso_backend/apps/core/gis/parsers.py index 16b771dd8..881c3f189 100644 --- a/terraso_backend/apps/core/gis/parsers.py +++ b/terraso_backend/apps/core/gis/parsers.py @@ -15,6 +15,7 @@ import json import os +import shutil import uuid import zipfile @@ -157,11 +158,7 @@ def parse_shapefile(file): gdf_transformed = gdf.to_crs(crs=DEFAULT_CRS) # Delete extracted files - for root, dirs, files in os.walk(tmp_folder, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) + shutil.rmtree(tmp_folder) return json.loads(gdf_transformed.to_json())