From c3c0bb994db125f2252c209885c716f86cdeabb2 Mon Sep 17 00:00:00 2001 From: Ted Wong Date: Fri, 23 Aug 2024 03:50:10 -0400 Subject: [PATCH 1/9] create data layer for worldpop_age_sex --- city_metrix/layers/worldpop_age_sex_class.py | 33 ++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 city_metrix/layers/worldpop_age_sex_class.py diff --git a/city_metrix/layers/worldpop_age_sex_class.py b/city_metrix/layers/worldpop_age_sex_class.py new file mode 100644 index 0000000..539d3d8 --- /dev/null +++ b/city_metrix/layers/worldpop_age_sex_class.py @@ -0,0 +1,33 @@ +from dask.diagnostics import ProgressBar +import xarray as xr +import xee +import ee + +from .layer import Layer, get_utm_zone_epsg, get_image_collection + + +class WorldPop(Layer): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.agesex_classes = agesex_classes + + def get_data(self, bbox): + # load population + dataset = ee.ImageCollection('WorldPop/GP/100m/pop_age_sex') + world_pop = ee.ImageCollection(dataset + .filterBounds(ee.Geometry.BBox(*bbox)) + .filter(ee.Filter.inList('year', [2020])) + .select(self.agesex_classes[0]) + ) + for agesex_class in self.agesex_classes[1:]: + world_pop = world_pop.add( + ee.ImageCollection(dataset + .filterBounds(ee.Geometry.BBox(*bbox)) + .filter(ee.Filter.inList('year', [2020])) + .select(agesex_class) + ) + ) + world_pop = world_pop.mean() # Should this be sum? + + data = get_image_collection(world_pop, bbox, 100, "world pop") + return data.population From f4bb00ae77bba67659a76e90d67ba6c5cef341c8 Mon Sep 17 00:00:00 2001 From: Ted Wong Date: Sun, 25 Aug 2024 19:54:59 -0400 Subject: [PATCH 2/9] change mean() to sum() in world_pop and world_pop_agesex --- city_metrix/layers/world_pop.py | 2 +- city_metrix/layers/worldpop_age_sex_class.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/city_metrix/layers/world_pop.py b/city_metrix/layers/world_pop.py index f3cf1a2..e9459f3 100644 --- a/city_metrix/layers/world_pop.py +++ b/city_metrix/layers/world_pop.py @@ -17,7 +17,7 @@ def get_data(self, bbox): .filterBounds(ee.Geometry.BBox(*bbox)) .filter(ee.Filter.inList('year', [2020])) .select('population') - .mean() + .sum() ) data = get_image_collection(world_pop, bbox, 100, "world pop") diff --git a/city_metrix/layers/worldpop_age_sex_class.py b/city_metrix/layers/worldpop_age_sex_class.py index 539d3d8..d0b44bd 100644 --- a/city_metrix/layers/worldpop_age_sex_class.py +++ b/city_metrix/layers/worldpop_age_sex_class.py @@ -27,7 +27,7 @@ def get_data(self, bbox): .select(agesex_class) ) ) - world_pop = world_pop.mean() # Should this be sum? + world_pop = world_pop.sum() data = get_image_collection(world_pop, bbox, 100, "world pop") return data.population From 20f6a103ed8650ca4ef12774f9853e3c8799fec5 Mon Sep 17 00:00:00 2001 From: Ted Wong Date: Mon, 16 Sep 2024 15:34:33 -0400 Subject: [PATCH 3/9] added test, input params, new class name --- city_metrix/layers/__init__.py | 1 + city_metrix/layers/worldpop_age_sex_class.py | 9 ++-- tests/test_layers.py | 49 +++++++++++--------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/city_metrix/layers/__init__.py b/city_metrix/layers/__init__.py index ea7fa8a..9f94ef7 100644 --- a/city_metrix/layers/__init__.py +++ b/city_metrix/layers/__init__.py @@ -11,6 +11,7 @@ from .landsat_collection_2 import LandsatCollection2 from .sentinel_2_level_2 import Sentinel2Level2 from .world_pop import WorldPop +from .world_pop_age_sex_class import WorldPopAgeSex from .built_up_height import BuiltUpHeight from .average_net_building_height import AverageNetBuildingHeight from .open_buildings import OpenBuildings diff --git a/city_metrix/layers/worldpop_age_sex_class.py b/city_metrix/layers/worldpop_age_sex_class.py index d0b44bd..f10623e 100644 --- a/city_metrix/layers/worldpop_age_sex_class.py +++ b/city_metrix/layers/worldpop_age_sex_class.py @@ -6,24 +6,25 @@ from .layer import Layer, get_utm_zone_epsg, get_image_collection -class WorldPop(Layer): - def __init__(self, **kwargs): +class WorldPopAgeSex(Layer): + def __init__(self, agesex_classes, year=2020, **kwargs): super().__init__(**kwargs) self.agesex_classes = agesex_classes + self.year = year def get_data(self, bbox): # load population dataset = ee.ImageCollection('WorldPop/GP/100m/pop_age_sex') world_pop = ee.ImageCollection(dataset .filterBounds(ee.Geometry.BBox(*bbox)) - .filter(ee.Filter.inList('year', [2020])) + .filter(ee.Filter.inList('year', [self.year])) .select(self.agesex_classes[0]) ) for agesex_class in self.agesex_classes[1:]: world_pop = world_pop.add( ee.ImageCollection(dataset .filterBounds(ee.Geometry.BBox(*bbox)) - .filter(ee.Filter.inList('year', [2020])) + .filter(ee.Filter.inList('year', [self.year])) .select(agesex_class) ) ) diff --git a/tests/test_layers.py b/tests/test_layers.py index 8372614..a90552e 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -21,30 +21,31 @@ TreeCanopyHeight, TreeCover, UrbanLandUse, - WorldPop + WorldPop, + WorldPopAgeSex ) from city_metrix.layers.layer import get_image_collection -from tests.fixtures.bbox_constants import BBOX_BRAZIL_LAURO_DE_FREITAS_1 +from tests.fixtures.bbox_constants import BBOX EE_IMAGE_DIMENSION_TOLERANCE = 1 # Tolerance compensates for variable results from GEE service def test_albedo(): - assert Albedo().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + assert Albedo().get_data(BBOX).mean() def test_alos_dsm(): - mean = AlosDSM().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + mean = AlosDSM().get_data(BBOX).mean() assert mean def test_average_net_building_height(): - assert AverageNetBuildingHeight().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + assert AverageNetBuildingHeight().get_data(BBOX).mean() def test_esa_world_cover(): count = ( EsaWorldCover(land_cover_class=EsaWorldCoverClass.BUILT_UP) - .get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + .get_data(BBOX) .count() ) assert count @@ -52,7 +53,7 @@ def test_esa_world_cover(): def test_read_image_collection(): ic = ee.ImageCollection("ESA/WorldCover/v100") - data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 10, "test") + data = get_image_collection(ic, BBOX, 10, "test") expected_crs = 32724 expected_x_dimension = 187 @@ -67,47 +68,47 @@ def test_read_image_collection(): def test_read_image_collection_scale(): ic = ee.ImageCollection("ESA/WorldCover/v100") - data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 100, "test") + data = get_image_collection(ic, BBOX, 100, "test") expected_x_dimension = 19 expected_y_dimension = 20 assert data.dims == {"x": expected_x_dimension, "y": expected_y_dimension} def test_high_land_surface_temperature(): - data = HighLandSurfaceTemperature().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + data = HighLandSurfaceTemperature().get_data(BBOX) assert data.any() def test_land_surface_temperature(): - mean_lst = LandSurfaceTemperature().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + mean_lst = LandSurfaceTemperature().get_data(BBOX).mean() assert mean_lst def test_landsat_collection_2(): bands = ['green'] - data = LandsatCollection2(bands).get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + data = LandsatCollection2(bands).get_data(BBOX) assert data.any() def test_nasa_dem(): - mean = NasaDEM().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + mean = NasaDEM().get_data(BBOX).mean() assert mean def test_natural_areas(): - data = NaturalAreas().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + data = NaturalAreas().get_data(BBOX) assert data.any() def test_openbuildings(): - count = OpenBuildings().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count().sum() + count = OpenBuildings().get_data(BBOX).count().sum() assert count def test_open_street_map(): count = ( OpenStreetMap(osm_class=OpenStreetMapClass.ROAD) - .get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + .get_data(BBOX) .count() .sum() ) @@ -115,28 +116,28 @@ def test_open_street_map(): def test_overture_buildings(): - count = OvertureBuildings().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count().sum() + count = OvertureBuildings().get_data(BBOX).count().sum() assert count def test_sentinel_2_level2(): sentinel_2_bands = ["green"] - data = Sentinel2Level2(sentinel_2_bands).get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + data = Sentinel2Level2(sentinel_2_bands).get_data(BBOX) assert data.any() def test_smart_surface_lulc(): - count = SmartSurfaceLULC().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() + count = SmartSurfaceLULC().get_data(BBOX).count() assert count def test_tree_canopy_height(): - count = TreeCanopyHeight().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() + count = TreeCanopyHeight().get_data(BBOX).count() assert count def test_tree_cover(): - actual = TreeCover().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + actual = TreeCover().get_data(BBOX).mean() expected = 54.0 tolerance = 0.1 assert ( @@ -145,9 +146,13 @@ def test_tree_cover(): def test_urban_land_use(): - assert UrbanLandUse().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() + assert UrbanLandUse().get_data(BBOX).count() def test_world_pop(): - data = WorldPop().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + data = WorldPop().get_data(BBOX) + assert data.any() + +def test_world_pop_age_sex(): + data = WorldPopAgeSex(agesex_classes=['M_70'] year=2020).get_data(BBOX) assert data.any() From 505151c3058d0a7fae8a5d8917d265f75e9e5abc Mon Sep 17 00:00:00 2001 From: Ted Wong Date: Mon, 16 Sep 2024 15:47:12 -0400 Subject: [PATCH 4/9] Update test_layers.py --- tests/test_layers.py | 140 +++++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 86 deletions(-) diff --git a/tests/test_layers.py b/tests/test_layers.py index a90552e..cb416ad 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -1,17 +1,19 @@ -import ee import pytest - +import numpy as np from city_metrix.layers import ( Albedo, AlosDSM, AverageNetBuildingHeight, + BuiltUpHeight, EsaWorldCover, EsaWorldCoverClass, HighLandSurfaceTemperature, + ImperviousSurface, LandsatCollection2, LandSurfaceTemperature, NasaDEM, NaturalAreas, + NdviSentinel2, OpenBuildings, OpenStreetMap, OpenStreetMapClass, @@ -24,135 +26,101 @@ WorldPop, WorldPopAgeSex ) -from city_metrix.layers.layer import get_image_collection -from tests.fixtures.bbox_constants import BBOX +from tests.resources.bbox_constants import BBOX_BRA_LAURO_DE_FREITAS_1 -EE_IMAGE_DIMENSION_TOLERANCE = 1 # Tolerance compensates for variable results from GEE service +# Tests are implemented for the same bounding box in Brazil. +COUNTRY_CODE_FOR_BBOX = 'BRA' +BBOX = BBOX_BRA_LAURO_DE_FREITAS_1 def test_albedo(): - assert Albedo().get_data(BBOX).mean() - + data = Albedo().get_data(BBOX) + assert np.size(data) > 0 def test_alos_dsm(): - mean = AlosDSM().get_data(BBOX).mean() - assert mean - + data = AlosDSM().get_data(BBOX) + assert np.size(data) > 0 def test_average_net_building_height(): - assert AverageNetBuildingHeight().get_data(BBOX).mean() + data = AverageNetBuildingHeight().get_data(BBOX) + assert np.size(data) > 0 +def test_built_up_height(): + data = BuiltUpHeight().get_data(BBOX) + assert np.size(data) > 0 def test_esa_world_cover(): - count = ( - EsaWorldCover(land_cover_class=EsaWorldCoverClass.BUILT_UP) - .get_data(BBOX) - .count() - ) - assert count - - -def test_read_image_collection(): - ic = ee.ImageCollection("ESA/WorldCover/v100") - data = get_image_collection(ic, BBOX, 10, "test") - - expected_crs = 32724 - expected_x_dimension = 187 - expected_y_dimension = 199 - - assert data.rio.crs == expected_crs - assert ( - pytest.approx(expected_x_dimension, rel=EE_IMAGE_DIMENSION_TOLERANCE) == "x", - pytest.approx(expected_y_dimension, rel=EE_IMAGE_DIMENSION_TOLERANCE) == "y" - ) - - -def test_read_image_collection_scale(): - ic = ee.ImageCollection("ESA/WorldCover/v100") - data = get_image_collection(ic, BBOX, 100, "test") - expected_x_dimension = 19 - expected_y_dimension = 20 - assert data.dims == {"x": expected_x_dimension, "y": expected_y_dimension} - + land_cover_class = EsaWorldCoverClass.BUILT_UP + data = EsaWorldCover(land_cover_class=land_cover_class).get_data(BBOX) + assert np.size(data) > 0 def test_high_land_surface_temperature(): data = HighLandSurfaceTemperature().get_data(BBOX) - assert data.any() + assert np.size(data) > 0 +def test_impervious_surface(): + data = ImperviousSurface().get_data(BBOX) + assert np.size(data) > 0 def test_land_surface_temperature(): - mean_lst = LandSurfaceTemperature().get_data(BBOX).mean() - assert mean_lst - + data = LandSurfaceTemperature().get_data(BBOX) + assert np.size(data) > 0 +@pytest.mark.skip(reason="layer is deprecated") def test_landsat_collection_2(): - bands = ['green'] + bands = ["blue"] data = LandsatCollection2(bands).get_data(BBOX) - assert data.any() - + assert np.size(data) > 0 def test_nasa_dem(): - mean = NasaDEM().get_data(BBOX).mean() - assert mean - + data = NasaDEM().get_data(BBOX) + assert np.size(data) > 0 def test_natural_areas(): data = NaturalAreas().get_data(BBOX) - assert data.any() + assert np.size(data) > 0 +def test_ndvi_sentinel2(): + data = NdviSentinel2(year=2023).get_data(BBOX) + assert np.size(data) > 0 def test_openbuildings(): - count = OpenBuildings().get_data(BBOX).count().sum() - assert count - + data = OpenBuildings(COUNTRY_CODE_FOR_BBOX).get_data(BBOX) + assert np.size(data) > 0 def test_open_street_map(): - count = ( - OpenStreetMap(osm_class=OpenStreetMapClass.ROAD) - .get_data(BBOX) - .count() - .sum() - ) - assert count - + data = OpenStreetMap(osm_class=OpenStreetMapClass.ROAD).get_data(BBOX) + assert np.size(data) > 0 def test_overture_buildings(): - count = OvertureBuildings().get_data(BBOX).count().sum() - assert count - + data = OvertureBuildings().get_data(BBOX) + assert np.size(data) > 0 +@pytest.mark.skip(reason="layer is deprecated") def test_sentinel_2_level2(): sentinel_2_bands = ["green"] data = Sentinel2Level2(sentinel_2_bands).get_data(BBOX) - assert data.any() - + assert np.size(data) > 0 def test_smart_surface_lulc(): - count = SmartSurfaceLULC().get_data(BBOX).count() - assert count - + data = SmartSurfaceLULC().get_data(BBOX) + assert np.size(data) > 0 def test_tree_canopy_height(): - count = TreeCanopyHeight().get_data(BBOX).count() - assert count - + data = TreeCanopyHeight().get_data(BBOX) + assert np.size(data) > 0 def test_tree_cover(): - actual = TreeCover().get_data(BBOX).mean() - expected = 54.0 - tolerance = 0.1 - assert ( - pytest.approx(expected, rel=tolerance) == actual - ) - + data = TreeCover().get_data(BBOX) + assert np.size(data) > 0 def test_urban_land_use(): - assert UrbanLandUse().get_data(BBOX).count() - + data = UrbanLandUse().get_data(BBOX) + assert np.size(data) > 0 def test_world_pop(): data = WorldPop().get_data(BBOX) - assert data.any() - + assert np.size(data) > 0 + def test_world_pop_age_sex(): data = WorldPopAgeSex(agesex_classes=['M_70'] year=2020).get_data(BBOX) - assert data.any() + assert data.any() \ No newline at end of file From 9a4e3fd49e0ec0a7d7b180d90dda5d7fc4b6d57e Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Wed, 18 Sep 2024 16:22:40 +0800 Subject: [PATCH 5/9] clean world pop age sex layer --- city_metrix/layers/__init__.py | 2 +- city_metrix/layers/world_pop_age_sex.py | 34 ++++++++++++++++++++ city_metrix/layers/worldpop_age_sex_class.py | 34 -------------------- tests/test_layer_parameters.py | 16 +++++++-- tests/test_layers.py | 4 +-- 5 files changed, 51 insertions(+), 39 deletions(-) create mode 100644 city_metrix/layers/world_pop_age_sex.py delete mode 100644 city_metrix/layers/worldpop_age_sex_class.py diff --git a/city_metrix/layers/__init__.py b/city_metrix/layers/__init__.py index e60e8b9..285db9f 100644 --- a/city_metrix/layers/__init__.py +++ b/city_metrix/layers/__init__.py @@ -12,7 +12,7 @@ from .landsat_collection_2 import LandsatCollection2 from .sentinel_2_level_2 import Sentinel2Level2 from .world_pop import WorldPop -from .world_pop_age_sex_class import WorldPopAgeSex +from .world_pop_age_sex import WorldPopAgeSex from .built_up_height import BuiltUpHeight from .average_net_building_height import AverageNetBuildingHeight from .open_buildings import OpenBuildings diff --git a/city_metrix/layers/world_pop_age_sex.py b/city_metrix/layers/world_pop_age_sex.py new file mode 100644 index 0000000..068abd6 --- /dev/null +++ b/city_metrix/layers/world_pop_age_sex.py @@ -0,0 +1,34 @@ +from dask.diagnostics import ProgressBar +import xarray as xr +import xee +import ee + +from .layer import Layer, get_utm_zone_epsg, get_image_collection + + +class WorldPopAgeSex(Layer): + """ + Attributes: + agesex_classes: list of age-sex classes to retrieve + year: year used for data retrieval + spatial_resolution: raster resolution in meters (see https://github.com/stac-extensions/raster) + """ + + def __init__(self, agesex_classes=['M_70'], year=2020, spatial_resolution=100, **kwargs): + super().__init__(**kwargs) + self.agesex_classes = agesex_classes + self.year = year + self.spatial_resolution = spatial_resolution + + def get_data(self, bbox): + # load population + dataset = ee.ImageCollection('WorldPop/GP/100m/pop_age_sex') + world_pop = ee.ImageCollection(dataset + .filterBounds(ee.Geometry.BBox(*bbox)) + .filter(ee.Filter.inList('year', [self.year])) + .select(self.agesex_classes) + .sum() + ) + + data = get_image_collection(world_pop, bbox, self.spatial_resolution, "world pop age sex") + return data.population diff --git a/city_metrix/layers/worldpop_age_sex_class.py b/city_metrix/layers/worldpop_age_sex_class.py deleted file mode 100644 index f10623e..0000000 --- a/city_metrix/layers/worldpop_age_sex_class.py +++ /dev/null @@ -1,34 +0,0 @@ -from dask.diagnostics import ProgressBar -import xarray as xr -import xee -import ee - -from .layer import Layer, get_utm_zone_epsg, get_image_collection - - -class WorldPopAgeSex(Layer): - def __init__(self, agesex_classes, year=2020, **kwargs): - super().__init__(**kwargs) - self.agesex_classes = agesex_classes - self.year = year - - def get_data(self, bbox): - # load population - dataset = ee.ImageCollection('WorldPop/GP/100m/pop_age_sex') - world_pop = ee.ImageCollection(dataset - .filterBounds(ee.Geometry.BBox(*bbox)) - .filter(ee.Filter.inList('year', [self.year])) - .select(self.agesex_classes[0]) - ) - for agesex_class in self.agesex_classes[1:]: - world_pop = world_pop.add( - ee.ImageCollection(dataset - .filterBounds(ee.Geometry.BBox(*bbox)) - .filter(ee.Filter.inList('year', [self.year])) - .select(agesex_class) - ) - ) - world_pop = world_pop.sum() - - data = get_image_collection(world_pop, bbox, 100, "world pop") - return data.population diff --git a/tests/test_layer_parameters.py b/tests/test_layer_parameters.py index 703752c..7902d99 100644 --- a/tests/test_layer_parameters.py +++ b/tests/test_layer_parameters.py @@ -8,18 +8,21 @@ AlosDSM, AverageNetBuildingHeight, BuiltUpHeight, - EsaWorldCover, EsaWorldCoverClass, + EsaWorldCover, + EsaWorldCoverClass, HighLandSurfaceTemperature, ImperviousSurface, LandSurfaceTemperature, NasaDEM, NaturalAreas, NdviSentinel2, + OpenBuildings, OpenStreetMap, TreeCanopyHeight, TreeCover, UrbanLandUse, - WorldPop, OpenBuildings + WorldPop, + WorldPopAgeSex ) from tests.resources.bbox_constants import BBOX_BRA_LAURO_DE_FREITAS_1 from tests.tools.general_tools import get_class_from_instance, get_class_default_spatial_resolution @@ -173,6 +176,15 @@ def test_world_pop_downsampled_spatial_resolution(self): assert pytest.approx(target_downsized_res, rel=RESOLUTION_TOLERANCE) == estimated_actual_res assert downsizing_is_within_tolerances + + def test_world_pop_age_sex_downsampled_spatial_resolution(self): + class_instance = WorldPopAgeSex() + default_res_data, downsized_res_data, target_downsized_res, estimated_actual_res = ( + _get_sample_data(class_instance, BBOX, DOWNSIZE_FACTOR)) + downsizing_is_within_tolerances = _evaluate_raster_value(default_res_data, downsized_res_data) + + assert pytest.approx(target_downsized_res, rel=RESOLUTION_TOLERANCE) == estimated_actual_res + assert downsizing_is_within_tolerances def test_halved_up_sampled_spatial_resolution(self): class_instance = Albedo() diff --git a/tests/test_layers.py b/tests/test_layers.py index 7d80c89..420a402 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -122,5 +122,5 @@ def test_world_pop(): assert np.size(data) > 0 def test_world_pop_age_sex(): - data = WorldPopAgeSex(agesex_classes=['M_70'], year=2020).get_data(BBOX) - assert np.size(data) > 0 \ No newline at end of file + data = WorldPopAgeSex().get_data(BBOX) + assert np.size(data) > 0 From ea890e4516bcdb8c6e94fa7a73ad055daaccbcb7 Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Wed, 18 Sep 2024 16:58:20 +0800 Subject: [PATCH 6/9] fix sum bands issue --- city_metrix/layers/world_pop.py | 8 +++++--- city_metrix/layers/world_pop_age_sex.py | 14 +++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/city_metrix/layers/world_pop.py b/city_metrix/layers/world_pop.py index d3b2aa6..a7e37d5 100644 --- a/city_metrix/layers/world_pop.py +++ b/city_metrix/layers/world_pop.py @@ -8,11 +8,13 @@ class WorldPop(Layer): """ Attributes: + year: year used for data retrieval spatial_resolution: raster resolution in meters (see https://github.com/stac-extensions/raster) """ - def __init__(self, spatial_resolution=100, **kwargs): + def __init__(self, year=2020, spatial_resolution=100, **kwargs): super().__init__(**kwargs) + self.year = year self.spatial_resolution = spatial_resolution def get_data(self, bbox): @@ -20,9 +22,9 @@ def get_data(self, bbox): dataset = ee.ImageCollection('WorldPop/GP/100m/pop') world_pop = ee.ImageCollection(dataset .filterBounds(ee.Geometry.BBox(*bbox)) - .filter(ee.Filter.inList('year', [2020])) + .filter(ee.Filter.inList('year', [self.year])) .select('population') - .sum() + .mean() ) data = get_image_collection(world_pop, bbox, self.spatial_resolution, "world pop") diff --git a/city_metrix/layers/world_pop_age_sex.py b/city_metrix/layers/world_pop_age_sex.py index 068abd6..ec3eda1 100644 --- a/city_metrix/layers/world_pop_age_sex.py +++ b/city_metrix/layers/world_pop_age_sex.py @@ -23,12 +23,12 @@ def __init__(self, agesex_classes=['M_70'], year=2020, spatial_resolution=100, * def get_data(self, bbox): # load population dataset = ee.ImageCollection('WorldPop/GP/100m/pop_age_sex') - world_pop = ee.ImageCollection(dataset - .filterBounds(ee.Geometry.BBox(*bbox)) - .filter(ee.Filter.inList('year', [self.year])) - .select(self.agesex_classes) - .sum() - ) + world_pop = dataset.filterBounds(ee.Geometry.BBox(*bbox))\ + .filter(ee.Filter.inList('year', [self.year]))\ + .select(self.agesex_classes)\ + .mean() + + world_pop = ee.ImageCollection(world_pop.reduce(ee.Reducer.sum()).rename('sum_age_sex_group')) data = get_image_collection(world_pop, bbox, self.spatial_resolution, "world pop age sex") - return data.population + return data.sum_age_sex_group From e9592c5562469b88dc52ed9908f5c063aa5e3974 Mon Sep 17 00:00:00 2001 From: Ted Wong Date: Mon, 23 Sep 2024 12:57:22 -0400 Subject: [PATCH 7/9] Total pop is sum over list of agesex classes --- city_metrix/layers/world_pop_age_sex.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/city_metrix/layers/world_pop_age_sex.py b/city_metrix/layers/world_pop_age_sex.py index ec3eda1..860642c 100644 --- a/city_metrix/layers/world_pop_age_sex.py +++ b/city_metrix/layers/world_pop_age_sex.py @@ -4,6 +4,7 @@ import ee from .layer import Layer, get_utm_zone_epsg, get_image_collection +from city_metrix.layers import WorldPop class WorldPopAgeSex(Layer): @@ -14,19 +15,27 @@ class WorldPopAgeSex(Layer): spatial_resolution: raster resolution in meters (see https://github.com/stac-extensions/raster) """ - def __init__(self, agesex_classes=['M_70'], year=2020, spatial_resolution=100, **kwargs): + def __init__(self, agesex_classes=[], year=2020, spatial_resolution=100, **kwargs): super().__init__(**kwargs) self.agesex_classes = agesex_classes self.year = year self.spatial_resolution = spatial_resolution def get_data(self, bbox): - # load population + # If no agesex classes requested, return plain WorldPop + if not self.agesex_classes: + return WorldPop().get_data(bbox) + dataset = ee.ImageCollection('WorldPop/GP/100m/pop_age_sex') - world_pop = dataset.filterBounds(ee.Geometry.BBox(*bbox))\ + + # Sum over agesex classes + world_pop = ee.Image(0) + for agesex_class in self.agesex_classes: + tmp = dataset.filterBounds(ee.Geometry.BBox(*bbox))\ .filter(ee.Filter.inList('year', [self.year]))\ - .select(self.agesex_classes)\ + .select(agesex_class)\ .mean() + world_pop = world_pop.add(tmp) world_pop = ee.ImageCollection(world_pop.reduce(ee.Reducer.sum()).rename('sum_age_sex_group')) From c010e553046beb4349dc208e14ea9bc7778a0a27 Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Wed, 25 Sep 2024 11:58:49 +0800 Subject: [PATCH 8/9] update world pop age sex group layer --- city_metrix/layers/__init__.py | 1 - city_metrix/layers/world_pop.py | 39 +++++++++++++++------- city_metrix/layers/world_pop_age_sex.py | 43 ------------------------- tests/test_layer_parameters.py | 12 +------ tests/test_layers.py | 7 +--- 5 files changed, 29 insertions(+), 73 deletions(-) delete mode 100644 city_metrix/layers/world_pop_age_sex.py diff --git a/city_metrix/layers/__init__.py b/city_metrix/layers/__init__.py index 285db9f..7e5e19e 100644 --- a/city_metrix/layers/__init__.py +++ b/city_metrix/layers/__init__.py @@ -12,7 +12,6 @@ from .landsat_collection_2 import LandsatCollection2 from .sentinel_2_level_2 import Sentinel2Level2 from .world_pop import WorldPop -from .world_pop_age_sex import WorldPopAgeSex from .built_up_height import BuiltUpHeight from .average_net_building_height import AverageNetBuildingHeight from .open_buildings import OpenBuildings diff --git a/city_metrix/layers/world_pop.py b/city_metrix/layers/world_pop.py index a7e37d5..d2f2f0f 100644 --- a/city_metrix/layers/world_pop.py +++ b/city_metrix/layers/world_pop.py @@ -5,27 +5,42 @@ from .layer import Layer, get_utm_zone_epsg, get_image_collection + class WorldPop(Layer): """ Attributes: + agesex_classes: list of age-sex classes to retrieve year: year used for data retrieval spatial_resolution: raster resolution in meters (see https://github.com/stac-extensions/raster) """ - def __init__(self, year=2020, spatial_resolution=100, **kwargs): + def __init__(self, agesex_classes=[], year=2020, spatial_resolution=100, **kwargs): super().__init__(**kwargs) + self.agesex_classes = agesex_classes self.year = year self.spatial_resolution = spatial_resolution def get_data(self, bbox): - # load population - dataset = ee.ImageCollection('WorldPop/GP/100m/pop') - world_pop = ee.ImageCollection(dataset - .filterBounds(ee.Geometry.BBox(*bbox)) - .filter(ee.Filter.inList('year', [self.year])) - .select('population') - .mean() - ) - - data = get_image_collection(world_pop, bbox, self.spatial_resolution, "world pop") - return data.population + if not self.agesex_classes: + # total population + dataset = ee.ImageCollection('WorldPop/GP/100m/pop') + world_pop = ee.ImageCollection(dataset + .filterBounds(ee.Geometry.BBox(*bbox)) + .filter(ee.Filter.inList('year', [self.year])) + .select('population') + .mean() + ) + + data = get_image_collection(world_pop, bbox, self.spatial_resolution, "world pop").population + + else: + # sum population for selected age-sex groups + dataset = ee.ImageCollection('WorldPop/GP/100m/pop_age_sex') + world_pop = dataset.filterBounds(ee.Geometry.BBox(*bbox))\ + .filter(ee.Filter.inList('year', [self.year]))\ + .select(self.agesex_classes)\ + .mean() + world_pop = ee.ImageCollection(world_pop.reduce(ee.Reducer.sum()).rename('sum_age_sex_group')) + data = get_image_collection(world_pop, bbox, self.spatial_resolution, "world pop age sex").sum_age_sex_group + + return data diff --git a/city_metrix/layers/world_pop_age_sex.py b/city_metrix/layers/world_pop_age_sex.py deleted file mode 100644 index 860642c..0000000 --- a/city_metrix/layers/world_pop_age_sex.py +++ /dev/null @@ -1,43 +0,0 @@ -from dask.diagnostics import ProgressBar -import xarray as xr -import xee -import ee - -from .layer import Layer, get_utm_zone_epsg, get_image_collection -from city_metrix.layers import WorldPop - - -class WorldPopAgeSex(Layer): - """ - Attributes: - agesex_classes: list of age-sex classes to retrieve - year: year used for data retrieval - spatial_resolution: raster resolution in meters (see https://github.com/stac-extensions/raster) - """ - - def __init__(self, agesex_classes=[], year=2020, spatial_resolution=100, **kwargs): - super().__init__(**kwargs) - self.agesex_classes = agesex_classes - self.year = year - self.spatial_resolution = spatial_resolution - - def get_data(self, bbox): - # If no agesex classes requested, return plain WorldPop - if not self.agesex_classes: - return WorldPop().get_data(bbox) - - dataset = ee.ImageCollection('WorldPop/GP/100m/pop_age_sex') - - # Sum over agesex classes - world_pop = ee.Image(0) - for agesex_class in self.agesex_classes: - tmp = dataset.filterBounds(ee.Geometry.BBox(*bbox))\ - .filter(ee.Filter.inList('year', [self.year]))\ - .select(agesex_class)\ - .mean() - world_pop = world_pop.add(tmp) - - world_pop = ee.ImageCollection(world_pop.reduce(ee.Reducer.sum()).rename('sum_age_sex_group')) - - data = get_image_collection(world_pop, bbox, self.spatial_resolution, "world pop age sex") - return data.sum_age_sex_group diff --git a/tests/test_layer_parameters.py b/tests/test_layer_parameters.py index 7902d99..3fe415b 100644 --- a/tests/test_layer_parameters.py +++ b/tests/test_layer_parameters.py @@ -21,8 +21,7 @@ TreeCanopyHeight, TreeCover, UrbanLandUse, - WorldPop, - WorldPopAgeSex + WorldPop ) from tests.resources.bbox_constants import BBOX_BRA_LAURO_DE_FREITAS_1 from tests.tools.general_tools import get_class_from_instance, get_class_default_spatial_resolution @@ -176,15 +175,6 @@ def test_world_pop_downsampled_spatial_resolution(self): assert pytest.approx(target_downsized_res, rel=RESOLUTION_TOLERANCE) == estimated_actual_res assert downsizing_is_within_tolerances - - def test_world_pop_age_sex_downsampled_spatial_resolution(self): - class_instance = WorldPopAgeSex() - default_res_data, downsized_res_data, target_downsized_res, estimated_actual_res = ( - _get_sample_data(class_instance, BBOX, DOWNSIZE_FACTOR)) - downsizing_is_within_tolerances = _evaluate_raster_value(default_res_data, downsized_res_data) - - assert pytest.approx(target_downsized_res, rel=RESOLUTION_TOLERANCE) == estimated_actual_res - assert downsizing_is_within_tolerances def test_halved_up_sampled_spatial_resolution(self): class_instance = Albedo() diff --git a/tests/test_layers.py b/tests/test_layers.py index 420a402..d26a917 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -23,8 +23,7 @@ TreeCanopyHeight, TreeCover, UrbanLandUse, - WorldPop, - WorldPopAgeSex + WorldPop ) from tests.resources.bbox_constants import BBOX_BRA_LAURO_DE_FREITAS_1 @@ -120,7 +119,3 @@ def test_urban_land_use(): def test_world_pop(): data = WorldPop().get_data(BBOX) assert np.size(data) > 0 - -def test_world_pop_age_sex(): - data = WorldPopAgeSex().get_data(BBOX) - assert np.size(data) > 0 From 8bd08bbb728a5c3fdfce09ac01d419a3cbed12b7 Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Fri, 18 Oct 2024 15:26:23 +0800 Subject: [PATCH 9/9] add age sex group doc link and comments --- city_metrix/layers/world_pop.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/city_metrix/layers/world_pop.py b/city_metrix/layers/world_pop.py index d2f2f0f..30fb6d8 100644 --- a/city_metrix/layers/world_pop.py +++ b/city_metrix/layers/world_pop.py @@ -9,13 +9,16 @@ class WorldPop(Layer): """ Attributes: - agesex_classes: list of age-sex classes to retrieve + agesex_classes: list of age-sex classes to retrieve (see https://airtable.com/appDWCVIQlVnLLaW2/tblYpXsxxuaOk3PaZ/viwExxAgTQKZnRfWU/recFjH7WngjltFMGi?blocks=hide) year: year used for data retrieval spatial_resolution: raster resolution in meters (see https://github.com/stac-extensions/raster) """ def __init__(self, agesex_classes=[], year=2020, spatial_resolution=100, **kwargs): super().__init__(**kwargs) + # agesex_classes options: + # M_0, M_1, M_5, M_10, M_15, M_20, M_25, M_30, M_35, M_40, M_45, M_50, M_55, M_60, M_65, M_70, M_75, M_80 + # F_0, F_1, F_5, F_10, F_15, F_20, F_25, F_30, F_35, F_40, F_45, F_50, F_55, F_60, F_65, F_70, F_75, F_80 self.agesex_classes = agesex_classes self.year = year self.spatial_resolution = spatial_resolution