From 9be92e3025e4da42b0d16f8882c8c08bca7b8148 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 8 Aug 2024 10:49:50 -0700 Subject: [PATCH 1/5] initial restructuring --- tests/test_layers.py | 120 ++++++++---------------------------------- tests/test_methods.py | 80 ++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 97 deletions(-) create mode 100644 tests/test_methods.py diff --git a/tests/test_layers.py b/tests/test_layers.py index 989c147..1312efe 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -19,82 +19,8 @@ TreeCover, UrbanLandUse, ) -from city_metrix.layers.layer import get_image_collection - -from .conftest import ( - LARGE_ZONES, - ZONES, - MockGroupByLayer, - MockLargeGroupByLayer, - MockLargeLayer, - MockLayer, - MockMaskLayer, -) -from .fixtures.bbox_constants import * - - -def test_count(): - counts = MockLayer().groupby(ZONES).count() - assert counts.size == 100 - assert all([count == 100 for count in counts]) - - -def test_mean(): - means = MockLayer().groupby(ZONES).mean() - assert means.size == 100 - assert all([mean == i for i, mean in enumerate(means)]) - - -def test_fishnetted_count(): - counts = MockLargeLayer().groupby(LARGE_ZONES).count() - assert counts.size == 100 - assert all([count == 100 for count in counts]) - - -def test_fishnetted_mean(): - means = MockLargeLayer().groupby(LARGE_ZONES).mean() - assert means.size == 100 - assert all([mean == i for i, mean in enumerate(means)]) - - -def test_masks(): - counts = MockLayer().mask(MockMaskLayer()).groupby(ZONES).count() - assert counts.size == 100 - for i, count in enumerate(counts): - if i % 2 == 0: - assert np.isnan(count) - else: - assert count == 100 - - -def test_group_by_layer(): - counts = MockLayer().groupby(ZONES, layer=MockGroupByLayer()).count() - assert all([count == {1: 50.0, 2: 50.0} for count in counts]) - - -def test_group_by_large_layer(): - counts = ( - MockLargeLayer().groupby(LARGE_ZONES, layer=MockLargeGroupByLayer()).count() - ) - assert all([count == {1: 50.0, 2: 50.0} for count in counts]) - - -def test_read_image_collection(): - ic = ee.ImageCollection("ESA/WorldCover/v100") - data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 10, "test") - - expected_crs = 32724 - expected_x_dimension = 187 - expected_y_dimension = 199 - - assert data.rio.crs == expected_crs - assert data.dims == {"x": expected_x_dimension, "y": expected_y_dimension} - - -def test_read_image_collection_scale(): - ic = ee.ImageCollection("ESA/WorldCover/v100") - data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 100, "test") - assert data.dims == {"x": 19, "y": 20} +from city_metrix.layers.esa_ndvi import EsaNdvi +from tests.fixtures.bbox_constants import BBOX_BRAZIL_LAURO_DE_FREITAS_1 def test_tree_cover(): @@ -102,7 +28,7 @@ def test_tree_cover(): expected = 54.0 tolerance = 0.1 assert ( - pytest.approx(expected, rel=tolerance) == actual + pytest.approx(expected, rel=tolerance) == actual ) @@ -110,11 +36,15 @@ def test_albedo(): assert Albedo().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() -def test_lst(): - mean = LandSurfaceTemperature().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() +def test_alos_dsm(): + mean = AlosDSM().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() assert mean +def test_average_net_building_height(): + assert AverageNetBuildingHeight().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + + def test_esa(): count = ( EsaWorldCover(land_cover_class=EsaWorldCoverClass.BUILT_UP) @@ -124,8 +54,14 @@ def test_esa(): assert count -def test_average_net_building_height(): - assert AverageNetBuildingHeight().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() +def test_lst(): + mean = LandSurfaceTemperature().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + assert mean + + +def test_nasa_dem(): + mean = NasaDEM().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + assert mean def test_open_street_map(): @@ -138,35 +74,25 @@ def test_open_street_map(): assert count -def test_urban_land_use(): - assert UrbanLandUse().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() - - def test_openbuildings(): count = OpenBuildings().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count().sum() assert count -def test_tree_canopy_hight(): - count = TreeCanopyHeight().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() +def test_overture_buildings(): + count = OvertureBuildings().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count().sum() assert count -def test_alos_dsm(): - mean = AlosDSM().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() - assert mean - - def test_smart_surface_lulc(): count = SmartSurfaceLULC().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() assert count -def test_overture_buildings(): - count = OvertureBuildings().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count().sum() +def test_tree_canopy_hight(): + count = TreeCanopyHeight().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() assert count -def test_nasa_dem(): - mean = NasaDEM().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() - assert mean +def test_urban_land_use(): + assert UrbanLandUse().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() diff --git a/tests/test_methods.py b/tests/test_methods.py new file mode 100644 index 0000000..2711677 --- /dev/null +++ b/tests/test_methods.py @@ -0,0 +1,80 @@ +import ee +import numpy as np +import pytest + +from city_metrix.layers.layer import get_image_collection + +from .conftest import ( + LARGE_ZONES, + ZONES, + MockGroupByLayer, + MockLargeGroupByLayer, + MockLargeLayer, + MockLayer, + MockMaskLayer, +) +from .fixtures.bbox_constants import * + + +def test_count(): + counts = MockLayer().groupby(ZONES).count() + assert counts.size == 100 + assert all([count == 100 for count in counts]) + + +def test_mean(): + means = MockLayer().groupby(ZONES).mean() + assert means.size == 100 + assert all([mean == i for i, mean in enumerate(means)]) + + +def test_fishnetted_count(): + counts = MockLargeLayer().groupby(LARGE_ZONES).count() + assert counts.size == 100 + assert all([count == 100 for count in counts]) + + +def test_fishnetted_mean(): + means = MockLargeLayer().groupby(LARGE_ZONES).mean() + assert means.size == 100 + assert all([mean == i for i, mean in enumerate(means)]) + + +def test_masks(): + counts = MockLayer().mask(MockMaskLayer()).groupby(ZONES).count() + assert counts.size == 100 + for i, count in enumerate(counts): + if i % 2 == 0: + assert np.isnan(count) + else: + assert count == 100 + + +def test_group_by_layer(): + counts = MockLayer().groupby(ZONES, layer=MockGroupByLayer()).count() + assert all([count == {1: 50.0, 2: 50.0} for count in counts]) + + +def test_group_by_large_layer(): + counts = ( + MockLargeLayer().groupby(LARGE_ZONES, layer=MockLargeGroupByLayer()).count() + ) + assert all([count == {1: 50.0, 2: 50.0} for count in counts]) + + +def test_read_image_collection(): + ic = ee.ImageCollection("ESA/WorldCover/v100") + data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 10, "test") + + expected_crs = 32724 + expected_x_dimension = 187 + expected_y_dimension = 199 + + assert data.rio.crs == expected_crs + assert data.dims == {"x": expected_x_dimension, "y": expected_y_dimension} + + +def test_read_image_collection_scale(): + ic = ee.ImageCollection("ESA/WorldCover/v100") + data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 100, "test") + assert data.dims == {"x": 19, "y": 20} From f594c97e2cf4da7a9432bf58768ba74c09700858 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Thu, 8 Aug 2024 16:48:11 -0700 Subject: [PATCH 2/5] Initial restructuring and addition of simple test for each layer. --- tests/test_layers.py | 71 ++++++++++++++++++++++++++++++------------- tests/test_methods.py | 2 -- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/tests/test_layers.py b/tests/test_layers.py index 1312efe..71266fe 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -1,5 +1,3 @@ -import ee -import numpy as np import pytest from city_metrix.layers import ( @@ -8,30 +6,25 @@ AverageNetBuildingHeight, EsaWorldCover, EsaWorldCoverClass, + HighLandSurfaceTemperature, + LandsatCollection2, LandSurfaceTemperature, NasaDEM, + NaturalAreas, OpenBuildings, OpenStreetMap, OpenStreetMapClass, OvertureBuildings, + Sentinel2Level2, SmartSurfaceLULC, TreeCanopyHeight, TreeCover, UrbanLandUse, + WorldPop ) -from city_metrix.layers.esa_ndvi import EsaNdvi from tests.fixtures.bbox_constants import BBOX_BRAZIL_LAURO_DE_FREITAS_1 -def test_tree_cover(): - actual = TreeCover().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() - expected = 54.0 - tolerance = 0.1 - assert ( - pytest.approx(expected, rel=tolerance) == actual - ) - - def test_albedo(): assert Albedo().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() @@ -45,7 +38,7 @@ def test_average_net_building_height(): assert AverageNetBuildingHeight().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() -def test_esa(): +def test_esa_world_cover(): count = ( EsaWorldCover(land_cover_class=EsaWorldCoverClass.BUILT_UP) .get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) @@ -54,9 +47,20 @@ def test_esa(): assert count -def test_lst(): - mean = LandSurfaceTemperature().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() - assert mean +def test_high_land_surface_temperature(): + data = HighLandSurfaceTemperature().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + assert data.any() + + +def test_land_surface_temperature(): + mean_lst = LandSurfaceTemperature().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + assert mean_lst + + +def test_landsat_collection_2(): + bands = ['green'] + data = LandsatCollection2(bands).get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + assert data.any() def test_nasa_dem(): @@ -64,6 +68,16 @@ def test_nasa_dem(): assert mean +def test_natural_areas(): + data = NaturalAreas().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + assert data.any() + + +def test_openbuildings(): + count = OpenBuildings().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count().sum() + assert count + + def test_open_street_map(): count = ( OpenStreetMap(osm_class=OpenStreetMapClass.ROAD) @@ -74,16 +88,17 @@ def test_open_street_map(): assert count -def test_openbuildings(): - count = OpenBuildings().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count().sum() - assert count - - def test_overture_buildings(): count = OvertureBuildings().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count().sum() assert count +def test_sentinal_2_level2(): + sentinal_2_bands = ["green"] + data = Sentinel2Level2(sentinal_2_bands).get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + assert data.any() + + def test_smart_surface_lulc(): count = SmartSurfaceLULC().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() assert count @@ -94,5 +109,19 @@ def test_tree_canopy_hight(): assert count +def test_tree_cover(): + actual = TreeCover().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + expected = 54.0 + tolerance = 0.1 + assert ( + pytest.approx(expected, rel=tolerance) == actual + ) + + def test_urban_land_use(): assert UrbanLandUse().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() + + +def test_world_pop(): + data = WorldPop().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + assert data.any() diff --git a/tests/test_methods.py b/tests/test_methods.py index 2711677..63c0885 100644 --- a/tests/test_methods.py +++ b/tests/test_methods.py @@ -1,9 +1,7 @@ import ee import numpy as np -import pytest from city_metrix.layers.layer import get_image_collection - from .conftest import ( LARGE_ZONES, ZONES, From 981852d3446736cf94e94eb2011d2bd7553f5652 Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Fri, 9 Aug 2024 11:03:06 -0700 Subject: [PATCH 3/5] Moved layer-specific tests into layers.py --- tests/test_layers.py | 20 ++++++++++++++++++++ tests/test_methods.py | 20 -------------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/test_layers.py b/tests/test_layers.py index 71266fe..a46a338 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -1,3 +1,4 @@ +import ee import pytest from city_metrix.layers import ( @@ -22,6 +23,7 @@ UrbanLandUse, WorldPop ) +from city_metrix.layers.layer import get_image_collection from tests.fixtures.bbox_constants import BBOX_BRAZIL_LAURO_DE_FREITAS_1 @@ -47,6 +49,24 @@ def test_esa_world_cover(): assert count +def test_read_image_collection(): + ic = ee.ImageCollection("ESA/WorldCover/v100") + data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 10, "test") + + expected_crs = 32724 + expected_x_dimension = 187 + expected_y_dimension = 199 + + assert data.rio.crs == expected_crs + assert data.dims == {"x": expected_x_dimension, "y": expected_y_dimension} + + +def test_read_image_collection_scale(): + ic = ee.ImageCollection("ESA/WorldCover/v100") + data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 100, "test") + assert data.dims == {"x": 19, "y": 20} + + def test_high_land_surface_temperature(): data = HighLandSurfaceTemperature().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) assert data.any() diff --git a/tests/test_methods.py b/tests/test_methods.py index 63c0885..4651883 100644 --- a/tests/test_methods.py +++ b/tests/test_methods.py @@ -1,7 +1,5 @@ -import ee import numpy as np -from city_metrix.layers.layer import get_image_collection from .conftest import ( LARGE_ZONES, ZONES, @@ -58,21 +56,3 @@ def test_group_by_large_layer(): MockLargeLayer().groupby(LARGE_ZONES, layer=MockLargeGroupByLayer()).count() ) assert all([count == {1: 50.0, 2: 50.0} for count in counts]) - - -def test_read_image_collection(): - ic = ee.ImageCollection("ESA/WorldCover/v100") - data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 10, "test") - - expected_crs = 32724 - expected_x_dimension = 187 - expected_y_dimension = 199 - - assert data.rio.crs == expected_crs - assert data.dims == {"x": expected_x_dimension, "y": expected_y_dimension} - - -def test_read_image_collection_scale(): - ic = ee.ImageCollection("ESA/WorldCover/v100") - data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 100, "test") - assert data.dims == {"x": 19, "y": 20} From 67957f3383c77a886dc00adc3d9ca3072edea14f Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Fri, 9 Aug 2024 13:25:07 -0700 Subject: [PATCH 4/5] Added image-dimension tolerance --- tests/test_layers.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/test_layers.py b/tests/test_layers.py index a46a338..8372614 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -26,6 +26,7 @@ from city_metrix.layers.layer import get_image_collection from tests.fixtures.bbox_constants import BBOX_BRAZIL_LAURO_DE_FREITAS_1 +EE_IMAGE_DIMENSION_TOLERANCE = 1 # Tolerance compensates for variable results from GEE service def test_albedo(): assert Albedo().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() @@ -58,13 +59,18 @@ def test_read_image_collection(): expected_y_dimension = 199 assert data.rio.crs == expected_crs - assert data.dims == {"x": expected_x_dimension, "y": expected_y_dimension} + assert ( + pytest.approx(expected_x_dimension, rel=EE_IMAGE_DIMENSION_TOLERANCE) == "x", + pytest.approx(expected_y_dimension, rel=EE_IMAGE_DIMENSION_TOLERANCE) == "y" + ) def test_read_image_collection_scale(): ic = ee.ImageCollection("ESA/WorldCover/v100") data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 100, "test") - assert data.dims == {"x": 19, "y": 20} + expected_x_dimension = 19 + expected_y_dimension = 20 + assert data.dims == {"x": expected_x_dimension, "y": expected_y_dimension} def test_high_land_surface_temperature(): @@ -113,9 +119,9 @@ def test_overture_buildings(): assert count -def test_sentinal_2_level2(): - sentinal_2_bands = ["green"] - data = Sentinel2Level2(sentinal_2_bands).get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) +def test_sentinel_2_level2(): + sentinel_2_bands = ["green"] + data = Sentinel2Level2(sentinel_2_bands).get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) assert data.any() @@ -124,7 +130,7 @@ def test_smart_surface_lulc(): assert count -def test_tree_canopy_hight(): +def test_tree_canopy_height(): count = TreeCanopyHeight().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() assert count From 6018aa823357dbb4ad0b8a1e319e621e95efe586 Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Tue, 13 Aug 2024 16:49:12 +0800 Subject: [PATCH 5/5] upgrade osmnx version --- city_metrix/layers/open_street_map.py | 7 ------- environment.yml | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/city_metrix/layers/open_street_map.py b/city_metrix/layers/open_street_map.py index a56ff6b..8a32936 100644 --- a/city_metrix/layers/open_street_map.py +++ b/city_metrix/layers/open_street_map.py @@ -54,10 +54,3 @@ def get_data(self, bbox): osm_feature = osm_feature.reset_index()[keep_col] return osm_feature - - def write(self, output_path): - self.data['bbox'] = str(self.data.total_bounds) - self.data['osm_class'] = str(self.osm_class.value) - - # Write to a GeoJSON file - self.data.to_file(output_path, driver='GeoJSON') diff --git a/environment.yml b/environment.yml index 7e05978..24ec040 100644 --- a/environment.yml +++ b/environment.yml @@ -13,7 +13,7 @@ dependencies: - xarray-spatial=0.3.7 - xee=0.0.3 - utm=0.7.0 - - osmnx=1.8.1 + - osmnx=1.9.0 - dask[complete]=2023.11.0 - matplotlib=3.8.2 - jupyterlab=4.0.10