From 29a0d8a4aca2fa39f32eba0a86f80a138ec6d830 Mon Sep 17 00:00:00 2001 From: Romain Hugonnet Date: Mon, 21 Oct 2024 18:32:14 -0800 Subject: [PATCH 1/8] Start sliderule pairing example --- doc/source/sliderule.md | 106 ++++++++++++++++++++++++++++++++++++++++ xdem/dem.py | 24 +++++++++ 2 files changed, 130 insertions(+) create mode 100644 doc/source/sliderule.md diff --git a/doc/source/sliderule.md b/doc/source/sliderule.md new file mode 100644 index 00000000..f1bd4702 --- /dev/null +++ b/doc/source/sliderule.md @@ -0,0 +1,106 @@ +--- +file_format: mystnb +mystnb: + execution_timeout: 60 +jupytext: + formats: md:myst + text_representation: + extension: .md + format_name: myst +kernelspec: + display_name: xdem-env + language: python + name: xdem +--- +(cheatsheet)= + +# Pair with SlideRule for reference + +Most analysis of **xDEM relies on independent, high-precision elevation data to use as reference**, whether for +coregistration, bias-corrections or uncertainty analysis. + +[SlideRule](https://slideruleearth.io/) provides the ideal way to retrieve such data by accessing big data archives +of high-precision elevations, such as ICESat-2 or GEDI, efficiently in the cloud. + +Below, a short example to coregister and perform uncertainty analysis of a DEM with ICESat-2 ATL06. + +```{code-cell} ipython3 +:tags: [remove-cell] + +# To get a good resolution for displayed figures +from matplotlib import pyplot +pyplot.rcParams['figure.dpi'] = 600 +pyplot.rcParams['savefig.dpi'] = 600 +pyplot.rcParams['font.size'] = 9 # Default 10 is a bit too big for coregistration plots +``` + +```{code-cell} ipython3 +from sliderule import sliderule, icesat2 +import geoutils as gu +import xdem +import numpy as np + +# Open example DEM +dem = xdem.DEM(xdem.examples.get_path("longyearbyen_ref_dem")) + +# Derive inlier mask as glaciers +glacier_outlines = gu.Vector(xdem.examples.get_path("longyearbyen_glacier_outlines")) +inlier_mask = ~glacier_outlines.create_mask(dem) + +# Define region of interest as DEM footprint +bounds = list(dem.get_bounds_projected(4326)) +region = sliderule.toregion(bounds)["poly"] + +# Initiliaze SlideRule client +sliderule.init("slideruleearth.io") + +# Define desired parameters for ICESat-2 ATL06 +params = { + "poly": region, + "srt": icesat2.SRT_LAND, # Surface-type + "cnf": icesat2.CNF_SURFACE_HIGH, # Confidence level + "ats": 20.0, # Minimum along-track spread + "cnt": 10, # Minimum count +} + +# Request ATL06 subsetting in parallel +gdf = icesat2.atl06sp(params) +gdf = gdf[np.isfinite(gdf["h_li"])] # Keep valid data +gdf = gdf[gdf["atl06_quality_summary"]==0] # Keep very high-confidence data +``` + +```{code-cell} ipython3 +# Run a translation coregistration +nk = xdem.coreg.NuthKaab() +aligned_dem = nk.fit_and_apply(reference_elev=gdf, to_be_aligned_elev=dem, inlier_mask=inlier_mask, z_name="h_li") +``` + +```{code-cell} ipython3 +:tags: [hide-input] +:mystnb: +: code_prompt_show: "Show plotting code" +: code_prompt_hide: "Hide plotting code" + +hs = dem.hillshade() +# Plot before and after +import matplotlib.pyplot as plt +gdf = gdf.to_crs(dem.crs) +f, ax = plt.subplots(1, 2) +ax[0].set_title("Before translations") +hs.plot(cmap="Greys_r", add_cbar=False) +z_pc = dem.interp_points((gdf.geometry.x.values, gdf.geometry.y.values)) +gdf["dh_bef"] = gdf["h_li"] - z_pc +gdf.plot(column="dh_bef", cmap='RdYlBu', vmin=-30, vmax=30, ax=ax[0], markersize=0.5) +ax[1].set_title("After translations") +hs.plot(cmap="Greys_r", add_cbar=False) +z_pc_aligned = aligned_dem.interp_points((gdf.geometry.x.values, gdf.geometry.y.values)) +gdf["dh_aft"] = gdf["h_li"] - z_pc_aligned +gdf.plot(column="dh_aft", cmap='RdYlBu', vmin=-30, vmax=30, ax=ax[1], markersize=0.5) +_ = ax[1].set_yticklabels([]) +plt.savefig("/home/atom/ongoing/test.png", dpi=400) +``` + +```{code-cell} ipython3 +# Run uncertainty analysis +sig_dem, rho_sig = aligned_dem.estimate_uncertainty(other_elev=gdf, z_name="h_li") +``` \ No newline at end of file diff --git a/xdem/dem.py b/xdem/dem.py index c235b897..3022825a 100644 --- a/xdem/dem.py +++ b/xdem/dem.py @@ -532,8 +532,32 @@ def estimate_uncertainty( :return: Uncertainty raster, Variogram of uncertainty correlation. """ +<<<<<<< Updated upstream # Elevation change dh = other_dem.reproject(self, silent=True) - self +======= + # Summarize approach steps + approach_dict = { + "H2022": {"heterosc": True, "multi_range": True}, + "R2009": {"heterosc": False, "multi_range": True}, + "Basic": {"heterosc": False, "multi_range": False}, + } + + # Stable terrain depending on input + if stable_terrain is None: + stable_terrain = np.ones(self.shape, dtype="uint8") + + # Elevation change with the other DEM or elevation point cloud + if isinstance(other_elev, DEM): + dh = other_elev.reproject(self, silent=True) - self + elif isinstance(other_elev, gpd.GeoDataFrame): + other_elev = other_elev.to_crs(self.crs) + points = (other_elev.geometry.x.values, other_elev.geometry.y.values) + dh = other_elev[z_name].values - self.interp_points(points) + stable_terrain = stable_terrain.interp_points(points, method="nearest") + else: + raise TypeError("Other elevation should be a DEM or elevation point cloud object.") +>>>>>>> Stashed changes # If the precision of the other DEM is the same, divide the dh values by sqrt(2) # See Equation 7 and 8 of Hugonnet et al. (2022) From d09cd68289296168db1f5722a3a7415d38f60d0b Mon Sep 17 00:00:00 2001 From: Romain Hugonnet Date: Thu, 5 Dec 2024 13:49:39 -0900 Subject: [PATCH 2/8] Finalize coregistration section of example --- dev-environment.yml | 1 + doc/source/sliderule.md | 34 ++++++++++++++++++++-------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/dev-environment.yml b/dev-environment.yml index 73eddcd5..3bb0093a 100644 --- a/dev-environment.yml +++ b/dev-environment.yml @@ -48,6 +48,7 @@ dependencies: - graphviz - myst-nb - numpydoc + - sliderule # To run pairing example - pip: # Optional dependency diff --git a/doc/source/sliderule.md b/doc/source/sliderule.md index f1bd4702..a5d70101 100644 --- a/doc/source/sliderule.md +++ b/doc/source/sliderule.md @@ -42,6 +42,8 @@ import numpy as np # Open example DEM dem = xdem.DEM(xdem.examples.get_path("longyearbyen_ref_dem")) +dem.set_vcrs("EGM96") +dem = dem.to_vcrs("Ellipsoid") # Derive inlier mask as glaciers glacier_outlines = gu.Vector(xdem.examples.get_path("longyearbyen_glacier_outlines")) @@ -81,26 +83,30 @@ aligned_dem = nk.fit_and_apply(reference_elev=gdf, to_be_aligned_elev=dem, inlie : code_prompt_show: "Show plotting code" : code_prompt_hide: "Hide plotting code" +# Derive hillshade for background hs = dem.hillshade() +# Convert to same CRS +gdf = gdf.to_crs(dem.crs) +vect = gu.Vector(gdf) +# Mask areas not in inlier mask (glaciers here) +pc_mask = inlier_mask.astype("uint8").interp_points((gdf.geometry.x.values, gdf.geometry.y.values), method="nearest") +vect.ds = vect.ds[pc_mask == 1] +# Get point differences before and after +z_pc = dem.interp_points((vect.ds.geometry.x.values, vect.ds.geometry.y.values)) +vect.ds["dh_bef"] = vect.ds["h_li"] - z_pc +z_pc_aligned = aligned_dem.interp_points((vect.ds.geometry.x.values, vect.ds.geometry.y.values)) +vect.ds["dh_aft"] = vect.ds["h_li"] - z_pc_aligned + # Plot before and after import matplotlib.pyplot as plt -gdf = gdf.to_crs(dem.crs) f, ax = plt.subplots(1, 2) -ax[0].set_title("Before translations") +ax[0].set_title("Before\ncoregistration") hs.plot(cmap="Greys_r", add_cbar=False) -z_pc = dem.interp_points((gdf.geometry.x.values, gdf.geometry.y.values)) -gdf["dh_bef"] = gdf["h_li"] - z_pc -gdf.plot(column="dh_bef", cmap='RdYlBu', vmin=-30, vmax=30, ax=ax[0], markersize=0.5) -ax[1].set_title("After translations") +vect.plot(column="dh_bef", cmap='RdYlBu', vmin=-10, vmax=10, ax=ax[0], markersize=0.5, cbar_title="Elevation differences (m)") +ax[1].set_title("After\ncoregistration") hs.plot(cmap="Greys_r", add_cbar=False) -z_pc_aligned = aligned_dem.interp_points((gdf.geometry.x.values, gdf.geometry.y.values)) -gdf["dh_aft"] = gdf["h_li"] - z_pc_aligned -gdf.plot(column="dh_aft", cmap='RdYlBu', vmin=-30, vmax=30, ax=ax[1], markersize=0.5) +vect.plot(column="dh_aft", cmap='RdYlBu', vmin=-10, vmax=10, ax=ax[1], markersize=0.5, legend=True, cbar_title="Elevation differences (m)") _ = ax[1].set_yticklabels([]) +plt.tight_layout() plt.savefig("/home/atom/ongoing/test.png", dpi=400) -``` - -```{code-cell} ipython3 -# Run uncertainty analysis -sig_dem, rho_sig = aligned_dem.estimate_uncertainty(other_elev=gdf, z_name="h_li") ``` \ No newline at end of file From 9add285aa051cf74d99732e72668ef300d9039c4 Mon Sep 17 00:00:00 2001 From: Romain Hugonnet Date: Thu, 5 Dec 2024 13:52:55 -0900 Subject: [PATCH 3/8] Linting --- doc/source/sliderule.md | 8 ++++---- xdem/dem.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/sliderule.md b/doc/source/sliderule.md index a5d70101..dca3e16f 100644 --- a/doc/source/sliderule.md +++ b/doc/source/sliderule.md @@ -16,10 +16,10 @@ kernelspec: # Pair with SlideRule for reference -Most analysis of **xDEM relies on independent, high-precision elevation data to use as reference**, whether for +Most analysis of **xDEM relies on independent, high-precision elevation data to use as reference**, whether for coregistration, bias-corrections or uncertainty analysis. -[SlideRule](https://slideruleearth.io/) provides the ideal way to retrieve such data by accessing big data archives +[SlideRule](https://slideruleearth.io/) provides the ideal way to retrieve such data by accessing big data archives of high-precision elevations, such as ICESat-2 or GEDI, efficiently in the cloud. Below, a short example to coregister and perform uncertainty analysis of a DEM with ICESat-2 ATL06. @@ -53,7 +53,7 @@ inlier_mask = ~glacier_outlines.create_mask(dem) bounds = list(dem.get_bounds_projected(4326)) region = sliderule.toregion(bounds)["poly"] -# Initiliaze SlideRule client +# Initialize SlideRule client sliderule.init("slideruleearth.io") # Define desired parameters for ICESat-2 ATL06 @@ -109,4 +109,4 @@ vect.plot(column="dh_aft", cmap='RdYlBu', vmin=-10, vmax=10, ax=ax[1], markersiz _ = ax[1].set_yticklabels([]) plt.tight_layout() plt.savefig("/home/atom/ongoing/test.png", dpi=400) -``` \ No newline at end of file +``` diff --git a/xdem/dem.py b/xdem/dem.py index 441d08e2..17f31bef 100644 --- a/xdem/dem.py +++ b/xdem/dem.py @@ -490,7 +490,7 @@ def coregister_3d( def estimate_uncertainty( self, other_elev: DEM | gpd.GeoDataFrame, - stable_terrain: Mask | NDArrayb = None, + stable_terrain: Mask = None, approach: Literal["H2022", "R2009", "Basic"] = "H2022", precision_of_other: Literal["finer"] | Literal["same"] = "finer", spread_estimator: Callable[[NDArrayf], np.floating[Any]] = nmad, @@ -542,7 +542,7 @@ def estimate_uncertainty( # Stable terrain depending on input if stable_terrain is None: - stable_terrain = np.ones(self.shape, dtype="uint8") + stable_terrain = self.copy(new_array=np.ones(self.shape, dtype=bool)) # Elevation change with the other DEM or elevation point cloud if isinstance(other_elev, DEM): From e0093aefdcc0a678fb9529bab8b1abeb28cbbbb8 Mon Sep 17 00:00:00 2001 From: Romain Hugonnet Date: Thu, 5 Dec 2024 14:47:53 -0900 Subject: [PATCH 4/8] Fix errors --- doc/source/sliderule.md | 13 ++++++++++--- xdem/dem.py | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/source/sliderule.md b/doc/source/sliderule.md index dca3e16f..45459b85 100644 --- a/doc/source/sliderule.md +++ b/doc/source/sliderule.md @@ -1,7 +1,7 @@ --- file_format: mystnb mystnb: - execution_timeout: 60 + execution_timeout: 150 jupytext: formats: md:myst text_representation: @@ -14,7 +14,7 @@ kernelspec: --- (cheatsheet)= -# Pair with SlideRule for reference +# Elevation reference with SlideRule Most analysis of **xDEM relies on independent, high-precision elevation data to use as reference**, whether for coregistration, bias-corrections or uncertainty analysis. @@ -75,6 +75,9 @@ gdf = gdf[gdf["atl06_quality_summary"]==0] # Keep very high-confidence data # Run a translation coregistration nk = xdem.coreg.NuthKaab() aligned_dem = nk.fit_and_apply(reference_elev=gdf, to_be_aligned_elev=dem, inlier_mask=inlier_mask, z_name="h_li") + +# Print the estimated translation parameters +print([k+f': {nk.meta["outputs"]["affine"][k]:.2f} meters' for k in ["shift_x", "shift_y", "shift_z"]]) ``` ```{code-cell} ipython3 @@ -108,5 +111,9 @@ hs.plot(cmap="Greys_r", add_cbar=False) vect.plot(column="dh_aft", cmap='RdYlBu', vmin=-10, vmax=10, ax=ax[1], markersize=0.5, legend=True, cbar_title="Elevation differences (m)") _ = ax[1].set_yticklabels([]) plt.tight_layout() -plt.savefig("/home/atom/ongoing/test.png", dpi=400) +``` + +```{code-cell} ipython3 +# Show full coregistration summary +nk.info() ``` diff --git a/xdem/dem.py b/xdem/dem.py index 17f31bef..458f24ca 100644 --- a/xdem/dem.py +++ b/xdem/dem.py @@ -547,6 +547,7 @@ def estimate_uncertainty( # Elevation change with the other DEM or elevation point cloud if isinstance(other_elev, DEM): dh = other_elev.reproject(self, silent=True) - self + stable_terrain = stable_terrain.data elif isinstance(other_elev, gpd.GeoDataFrame): other_elev = other_elev.to_crs(self.crs) points = (other_elev.geometry.x.values, other_elev.geometry.y.values) From d5cade7b30cea68a9124160ffd3e69cbf3414f82 Mon Sep 17 00:00:00 2001 From: Romain Hugonnet Date: Thu, 5 Dec 2024 15:28:15 -0900 Subject: [PATCH 5/8] Add to index --- doc/source/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/index.md b/doc/source/index.md index d2be2649..d49c2f06 100644 --- a/doc/source/index.md +++ b/doc/source/index.md @@ -123,6 +123,7 @@ uncertainty guides cheatsheet +sliderule ecosystem ``` From 71ed3e53f0658ae6690cb414b9b0e1aedfd65331 Mon Sep 17 00:00:00 2001 From: Romain Hugonnet Date: Thu, 5 Dec 2024 16:50:17 -0900 Subject: [PATCH 6/8] Incremental changes --- doc/source/conf.py | 1 + doc/source/sliderule.md | 31 ++++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index c8d5dd7a..19df486e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -79,6 +79,7 @@ "geopandas": ("https://geopandas.org/en/stable", None), "xarray": ("https://docs.xarray.dev/en/stable/", None), "rioxarray": ("https://corteva.github.io/rioxarray/stable/", None), + "sliderule": ("https://slideruleearth.io/web/rtd/", None), } sphinx_gallery_conf = { diff --git a/doc/source/sliderule.md b/doc/source/sliderule.md index 45459b85..e38eb365 100644 --- a/doc/source/sliderule.md +++ b/doc/source/sliderule.md @@ -14,7 +14,7 @@ kernelspec: --- (cheatsheet)= -# Elevation reference with SlideRule +# Pair with reference data Most analysis of **xDEM relies on independent, high-precision elevation data to use as reference**, whether for coregistration, bias-corrections or uncertainty analysis. @@ -22,7 +22,7 @@ coregistration, bias-corrections or uncertainty analysis. [SlideRule](https://slideruleearth.io/) provides the ideal way to retrieve such data by accessing big data archives of high-precision elevations, such as ICESat-2 or GEDI, efficiently in the cloud. -Below, a short example to coregister and perform uncertainty analysis of a DEM with ICESat-2 ATL06. +Below, a short example to coregister our example DEM with ICESat-2 ATL06 reference data. ```{code-cell} ipython3 :tags: [remove-cell] @@ -34,6 +34,10 @@ pyplot.rcParams['savefig.dpi'] = 600 pyplot.rcParams['font.size'] = 9 # Default 10 is a bit too big for coregistration plots ``` +## Retrieving reference data + +We load our example DEM data, and define the region of interest from its footprint. + ```{code-cell} ipython3 from sliderule import sliderule, icesat2 import geoutils as gu @@ -52,7 +56,11 @@ inlier_mask = ~glacier_outlines.create_mask(dem) # Define region of interest as DEM footprint bounds = list(dem.get_bounds_projected(4326)) region = sliderule.toregion(bounds)["poly"] +``` +We can then initialize the SlideRule client, and fetch the ICESat-2 ATL06 reference data using {func}`icesat2.atl06sp`. + +```{code-cell} ipython3 # Initialize SlideRule client sliderule.init("slideruleearth.io") @@ -71,15 +79,22 @@ gdf = gdf[np.isfinite(gdf["h_li"])] # Keep valid data gdf = gdf[gdf["atl06_quality_summary"]==0] # Keep very high-confidence data ``` +## Coregistration + +Running the coregistration is as simple as passing the geodataframe to {func}`~xdem.coreg.Coreg.fit_and_apply`, +specifying the name of the column to use. + ```{code-cell} ipython3 # Run a translation coregistration nk = xdem.coreg.NuthKaab() -aligned_dem = nk.fit_and_apply(reference_elev=gdf, to_be_aligned_elev=dem, inlier_mask=inlier_mask, z_name="h_li") +aligned_dem = nk.fit_and_apply(reference_elev=gdf, to_be_aligned_elev=dem, inlier_mask=inlier_mask, z_name="h_li", random_state=42) # Print the estimated translation parameters print([k+f': {nk.meta["outputs"]["affine"][k]:.2f} meters' for k in ["shift_x", "shift_y", "shift_z"]]) ``` +We can plot the improvement due to the coregistration. + ```{code-cell} ipython3 :tags: [hide-input] :mystnb: @@ -96,9 +111,13 @@ pc_mask = inlier_mask.astype("uint8").interp_points((gdf.geometry.x.values, gdf. vect.ds = vect.ds[pc_mask == 1] # Get point differences before and after z_pc = dem.interp_points((vect.ds.geometry.x.values, vect.ds.geometry.y.values)) -vect.ds["dh_bef"] = vect.ds["h_li"] - z_pc z_pc_aligned = aligned_dem.interp_points((vect.ds.geometry.x.values, vect.ds.geometry.y.values)) -vect.ds["dh_aft"] = vect.ds["h_li"] - z_pc_aligned +import warnings +# GeoPandas raising unexpected warning +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + vect["dh_bef"] = vect["h_li"] - z_pc + vect["dh_aft"] = vect["h_li"] - z_pc_aligned # Plot before and after import matplotlib.pyplot as plt @@ -113,6 +132,8 @@ _ = ax[1].set_yticklabels([]) plt.tight_layout() ``` +And the full information around the coregistration using {func}`~xdem.coreg.Coreg.info`. + ```{code-cell} ipython3 # Show full coregistration summary nk.info() From e843376e783c56cd38dcaeda8a3ae6f7a9f04eaa Mon Sep 17 00:00:00 2001 From: Romain Hugonnet Date: Thu, 5 Dec 2024 17:00:18 -0900 Subject: [PATCH 7/8] Streamlining --- doc/source/sliderule.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/source/sliderule.md b/doc/source/sliderule.md index e38eb365..274aa857 100644 --- a/doc/source/sliderule.md +++ b/doc/source/sliderule.md @@ -58,7 +58,8 @@ bounds = list(dem.get_bounds_projected(4326)) region = sliderule.toregion(bounds)["poly"] ``` -We can then initialize the SlideRule client, and fetch the ICESat-2 ATL06 reference data using {func}`icesat2.atl06sp`. +We can then initialize the SlideRule client, and fetch the ICESat-2 ATL06 reference data in the region of interest +using {func}`icesat2.atl06sp`. ```{code-cell} ipython3 # Initialize SlideRule client @@ -93,7 +94,7 @@ aligned_dem = nk.fit_and_apply(reference_elev=gdf, to_be_aligned_elev=dem, inlie print([k+f': {nk.meta["outputs"]["affine"][k]:.2f} meters' for k in ["shift_x", "shift_y", "shift_z"]]) ``` -We can plot the improvement due to the coregistration. +Let's visualize the improvement in elevation differences after coregistration. ```{code-cell} ipython3 :tags: [hide-input] @@ -132,9 +133,9 @@ _ = ax[1].set_yticklabels([]) plt.tight_layout() ``` -And the full information around the coregistration using {func}`~xdem.coreg.Coreg.info`. +We can print a coregistration summary using {func}`~xdem.coreg.Coreg.info`. ```{code-cell} ipython3 -# Show full coregistration summary +# Show full coregistration information nk.info() ``` From 031e7612e492a00a7da559f301ac1eb81eca6b07 Mon Sep 17 00:00:00 2001 From: Romain Hugonnet Date: Thu, 5 Dec 2024 17:00:35 -0900 Subject: [PATCH 8/8] Linting --- doc/source/sliderule.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/sliderule.md b/doc/source/sliderule.md index 274aa857..88c58ea4 100644 --- a/doc/source/sliderule.md +++ b/doc/source/sliderule.md @@ -58,7 +58,7 @@ bounds = list(dem.get_bounds_projected(4326)) region = sliderule.toregion(bounds)["poly"] ``` -We can then initialize the SlideRule client, and fetch the ICESat-2 ATL06 reference data in the region of interest +We can then initialize the SlideRule client, and fetch the ICESat-2 ATL06 reference data in the region of interest using {func}`icesat2.atl06sp`. ```{code-cell} ipython3