From f40226bef77d8f7ad0b935f008405392c910e73a Mon Sep 17 00:00:00 2001 From: tsutterley Date: Thu, 30 May 2024 15:40:00 -0700 Subject: [PATCH 1/6] fix: single track plots for ATL08 variables --- clients/python/sliderule/io.py | 4 +++- clients/python/sliderule/ipysliderule.py | 29 ++++++++++++------------ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/clients/python/sliderule/io.py b/clients/python/sliderule/io.py index 4268120a4..1a4e4a6ae 100644 --- a/clients/python/sliderule/io.py +++ b/clients/python/sliderule/io.py @@ -482,8 +482,10 @@ def to_parquet(gdf, filename, **kwargs): ) # output metadata output = {} + # remove request polygon from output parameters + if 'poly' in kwargs['parameters']: + kwargs['parameters'].pop('poly') # for each adjustable sliderule parameter - [kwargs['parameters'].pop(p) for p in ['poly']] for p,val in kwargs['parameters'].items(): # try to convert the parameter if available try: diff --git a/clients/python/sliderule/ipysliderule.py b/clients/python/sliderule/ipysliderule.py index 54c3382d9..51f3622a4 100644 --- a/clients/python/sliderule/ipysliderule.py +++ b/clients/python/sliderule/ipysliderule.py @@ -2977,7 +2977,8 @@ def plot(self, **kwargs): elif (kwargs['kind'] == 'scatter'): # extract orbital cycle parameters cycle = int(kwargs['cycle']) - if (kwargs['data_type'] == 'atl03'): + data_type = kwargs['data_type'] + if (data_type == 'atl03'): # reduce entered data frame to RGT, ground track and cycle atl03 = self._gdf[(self._gdf['rgt'] == RGT) & (self._gdf['track'] == self.PT(GT)) & @@ -2987,11 +2988,11 @@ def plot(self, **kwargs): atl03['segment_dist'] += atl03['x_atc'] atl03['segment_dist'] -= kwargs['x_offset'] atl03.set_index('segment_dist', inplace=True) - if (kwargs['data_type'] == 'atl03') and (kwargs['classification'] == 'atl08'): + if (data_type == 'atl03') and (kwargs['classification'] == 'atl08'): # noise, ground, canopy, top of canopy, unclassified colormap = np.array(['c','b','limegreen','g','y']) classes = ['noise','ground','canopy','top of canopy','unclassified'] - sc = ax.scatter(atl03.index.values, atl03["height"].values, + sc = ax.scatter(atl03.index.values, atl03[column].values, c=colormap[atl03["atl08_class"].values.astype('i')], s=1.5, rasterized=True) # append handles to legend @@ -2999,20 +3000,20 @@ def plot(self, **kwargs): handle = matplotlib.lines.Line2D([0], [0], color=colormap[i], lw=6, label=lab) legend_handles.append(handle) - elif (kwargs['data_type'] == 'atl03') and (kwargs['classification'] == 'yapc'): + elif (data_type == 'atl03') and (kwargs['classification'] == 'yapc'): sc = ax.scatter(atl03.index.values, - atl03["height"].values, + atl03[column].values, c=atl03["yapc_score"], cmap=kwargs['cmap'], s=1.5, rasterized=True) plt.colorbar(sc) - elif (kwargs['data_type'] == 'atl03') and (kwargs['classification'] == 'atl03'): + elif (data_type == 'atl03') and (kwargs['classification'] == 'atl03'): # background, buffer, low, medium, high colormap = np.array(['y','c','b','g','m']) confidences = ['background','buffer','low','medium','high'] # reduce data frame to photon classified for surface atl03 = atl03[atl03["atl03_cnf"] >= 0] - sc = ax.scatter(atl03.index.values, atl03["height"].values, + sc = ax.scatter(atl03.index.values, atl03[column].values, c=colormap[atl03["atl03_cnf"].values.astype('i')], s=1.5, rasterized=True) # append handles to legend @@ -3020,16 +3021,16 @@ def plot(self, **kwargs): handle = matplotlib.lines.Line2D([0], [0], color=colormap[i], lw=6, label=lab) legend_handles.append(handle) - elif (kwargs['data_type'] == 'atl03'): + elif (data_type == 'atl03'): # plot all available ATL03 points as gray - sc = ax.scatter(atl03.index.values, atl03["height"].values, + sc = ax.scatter(atl03.index.values, atl03[column].values, c='0.4', s=0.5, rasterized=True) # append handle to legend handle = matplotlib.lines.Line2D([0], [0], color='0.4', lw=6, label='ATL03') legend_handles.append(handle) - # plot ATL06-SR segments for cycle and track - if (kwargs['data_type'] == 'atl06'): + # plot ATL06-SR or ATL08-SR segments for cycle and track + if data_type in ('atl06','atl08'): atl06 = self._gdf[ (self._gdf['rgt'] == RGT) & (self._gdf['gt'] == GT) & @@ -3039,14 +3040,14 @@ def plot(self, **kwargs): atl06.set_index('x_atc', inplace=True) # plot reduced data frame sc = ax.scatter(atl06.index.values, - atl06["h_mean"].values, + atl06[column].values, c='red', s=2.5, rasterized=True) handle = matplotlib.lines.Line2D([0], [0], - color='red', lw=6, label='ATL06-SR') + color='red', lw=6, label=f'{data_type.upper()}-SR') legend_handles.append(handle) # add axes labels ax.set_xlabel('Along-Track Distance [m]') - ax.set_ylabel('Height (m)') + ax.set_ylabel(f'SlideRule {column}') # add title if kwargs['title']: ax.set_title(kwargs['title']) From f8b9c5025c1fc7e94c7054b33a3055cff4788394 Mon Sep 17 00:00:00 2001 From: tsutterley Date: Mon, 3 Jun 2024 10:48:49 -0700 Subject: [PATCH 2/6] Update ipysliderule.py --- clients/python/sliderule/ipysliderule.py | 50 ++++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/clients/python/sliderule/ipysliderule.py b/clients/python/sliderule/ipysliderule.py index 51f3622a4..a85b82d4a 100644 --- a/clients/python/sliderule/ipysliderule.py +++ b/clients/python/sliderule/ipysliderule.py @@ -2949,22 +2949,22 @@ def plot(self, **kwargs): if (cycle < kwargs['cycle_start']): continue # reduce data frame to RGT, ground track and cycle - atl06 = self._gdf[ + gdf = self._gdf[ (self._gdf['rgt'] == RGT) & (self._gdf['gt'] == GT) & (self._gdf['cycle'] == cycle)] - if not any(atl06[column].values): + if not any(gdf[column].values): continue # set index to along-track distance - atl06['x_atc'] -= kwargs['x_offset'] - atl06.set_index('x_atc', inplace=True) + gdf['x_atc'] -= kwargs['x_offset'] + gdf.set_index('x_atc', inplace=True) # plot reduced data frame - l, = ax.plot(atl06.index.values, - atl06[column].values, + l, = ax.plot(gdf.index.values, + gdf[column].values, marker='.', lw=0, ms=1.5) # create legend element for cycle if (kwargs['legend_label'] == 'date'): - label = atl06.index[0].strftime('%Y-%m-%d') + label = gdf.index[0].strftime('%Y-%m-%d') elif (kwargs['legend_label'] == 'cycle'): label = f'Cycle {cycle:0.0f}' # append handle to legend @@ -2980,20 +2980,20 @@ def plot(self, **kwargs): data_type = kwargs['data_type'] if (data_type == 'atl03'): # reduce entered data frame to RGT, ground track and cycle - atl03 = self._gdf[(self._gdf['rgt'] == RGT) & + gdf = self._gdf[(self._gdf['rgt'] == RGT) & (self._gdf['track'] == self.PT(GT)) & (self._gdf['pair'] == self.LR(GT)) & (self._gdf['cycle'] == cycle)] # set index to along-track distance - atl03['segment_dist'] += atl03['x_atc'] - atl03['segment_dist'] -= kwargs['x_offset'] - atl03.set_index('segment_dist', inplace=True) + gdf['segment_dist'] += gdf['x_atc'] + gdf['segment_dist'] -= kwargs['x_offset'] + gdf.set_index('segment_dist', inplace=True) if (data_type == 'atl03') and (kwargs['classification'] == 'atl08'): # noise, ground, canopy, top of canopy, unclassified colormap = np.array(['c','b','limegreen','g','y']) classes = ['noise','ground','canopy','top of canopy','unclassified'] - sc = ax.scatter(atl03.index.values, atl03[column].values, - c=colormap[atl03["atl08_class"].values.astype('i')], + sc = ax.scatter(gdf.index.values, gdf[column].values, + c=colormap[gdf["atl08_class"].values.astype('i')], s=1.5, rasterized=True) # append handles to legend for i,lab in enumerate(classes): @@ -3001,9 +3001,9 @@ def plot(self, **kwargs): color=colormap[i], lw=6, label=lab) legend_handles.append(handle) elif (data_type == 'atl03') and (kwargs['classification'] == 'yapc'): - sc = ax.scatter(atl03.index.values, - atl03[column].values, - c=atl03["yapc_score"], + sc = ax.scatter(gdf.index.values, + gdf[column].values, + c=gdf["yapc_score"], cmap=kwargs['cmap'], s=1.5, rasterized=True) plt.colorbar(sc) @@ -3012,9 +3012,9 @@ def plot(self, **kwargs): colormap = np.array(['y','c','b','g','m']) confidences = ['background','buffer','low','medium','high'] # reduce data frame to photon classified for surface - atl03 = atl03[atl03["atl03_cnf"] >= 0] - sc = ax.scatter(atl03.index.values, atl03[column].values, - c=colormap[atl03["atl03_cnf"].values.astype('i')], + gdf = gdf[gdf["atl03_cnf"] >= 0] + sc = ax.scatter(gdf.index.values, gdf[column].values, + c=colormap[gdf["atl03_cnf"].values.astype('i')], s=1.5, rasterized=True) # append handles to legend for i,lab in enumerate(confidences): @@ -3023,7 +3023,7 @@ def plot(self, **kwargs): legend_handles.append(handle) elif (data_type == 'atl03'): # plot all available ATL03 points as gray - sc = ax.scatter(atl03.index.values, atl03[column].values, + sc = ax.scatter(gdf.index.values, gdf[column].values, c='0.4', s=0.5, rasterized=True) # append handle to legend handle = matplotlib.lines.Line2D([0], [0], @@ -3031,16 +3031,16 @@ def plot(self, **kwargs): legend_handles.append(handle) # plot ATL06-SR or ATL08-SR segments for cycle and track if data_type in ('atl06','atl08'): - atl06 = self._gdf[ + gdf = self._gdf[ (self._gdf['rgt'] == RGT) & (self._gdf['gt'] == GT) & (self._gdf['cycle'] == cycle)] # set index to along-track distance - atl06['x_atc'] -= kwargs['x_offset'] - atl06.set_index('x_atc', inplace=True) + gdf['x_atc'] -= kwargs['x_offset'] + gdf.set_index('x_atc', inplace=True) # plot reduced data frame - sc = ax.scatter(atl06.index.values, - atl06[column].values, + sc = ax.scatter(gdf.index.values, + gdf[column].values, c='red', s=2.5, rasterized=True) handle = matplotlib.lines.Line2D([0], [0], color='red', lw=6, label=f'{data_type.upper()}-SR') From 4b8c9186dc784bdae571c1e600f9f2f0ee66c943 Mon Sep 17 00:00:00 2001 From: tsutterley Date: Mon, 24 Jun 2024 17:48:36 -0700 Subject: [PATCH 3/6] add function to get version for a short_name --- clients/python/sliderule/earthdata.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/clients/python/sliderule/earthdata.py b/clients/python/sliderule/earthdata.py index d3b6c4927..15de6cef0 100644 --- a/clients/python/sliderule/earthdata.py +++ b/clients/python/sliderule/earthdata.py @@ -471,6 +471,25 @@ def __stac_search(provider, short_name, collections, polygons, time_start, time_ # return geojson dictionary return geojson +# +# Get Version from DATASETS +# +def __get_version(short_name): + + # check parameters + if short_name == None: + raise sliderule.FatalError("Must supply short name to CMR query") + elif short_name not in DATASETS: + raise sliderule.FatalError("Must supply a supported dataset: " + short_name) + + # attempt to fill in version + version = DATASETS[short_name]["version"] + if version == None: + raise sliderule.FatalError("Unable to determine version for CMR query") + + # return version string (cannot be None) + return version + # # Get Provider from DATASETS # From 9bc76ee3f412d31fb3cf2ec6a6bc674770820335 Mon Sep 17 00:00:00 2001 From: tsutterley Date: Mon, 24 Jun 2024 18:08:54 -0700 Subject: [PATCH 4/6] numpy 2.0 deprecated `np.float_` --- clients/python/sliderule/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/python/sliderule/io.py b/clients/python/sliderule/io.py index 1a4e4a6ae..34b92c90e 100644 --- a/clients/python/sliderule/io.py +++ b/clients/python/sliderule/io.py @@ -262,7 +262,7 @@ def attributes_encoder(attr): if isinstance(attr, (np.int_, np.intc, np.intp, np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64)): return int(attr) - elif isinstance(attr, (np.float_, np.float16, np.float32, np.float64)): + elif isinstance(attr, (np.float16, np.float32, np.float64)): return float(attr) elif isinstance(attr, (np.ndarray)): return attr.tolist() From 2a5f6041682d469cea13fc575e3770d25cbb4cd7 Mon Sep 17 00:00:00 2001 From: tsutterley Date: Tue, 25 Jun 2024 13:19:33 -0700 Subject: [PATCH 5/6] make draw control a map object attribute --- clients/python/sliderule/ipysliderule.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/python/sliderule/ipysliderule.py b/clients/python/sliderule/ipysliderule.py index a85b82d4a..b23f6af58 100644 --- a/clients/python/sliderule/ipysliderule.py +++ b/clients/python/sliderule/ipysliderule.py @@ -2026,17 +2026,17 @@ def __init__(self, projection, **kwargs): # keep track of cursor position self.map.on_interaction(self.handle_interaction) # add control for drawing polygons or bounding boxes - draw_control = ipyleaflet.DrawControl(polyline={},circlemarker={}, + self.draw_control = ipyleaflet.DrawControl(polyline={},circlemarker={}, edit=False) shapeOptions = {'color':kwargs['color'],'fill_color':kwargs['color']} - draw_control.rectangle = dict(shapeOptions=shapeOptions, + self.draw_control.rectangle = dict(shapeOptions=shapeOptions, metric=['km','m']) - draw_control.polygon = dict(shapeOptions=shapeOptions, + self.draw_control.polygon = dict(shapeOptions=shapeOptions, allowIntersection=False, showArea=True, metric=['km','m']) # create regions self.regions = [] - draw_control.on_draw(self.handle_draw) - self.map.add(draw_control) + self.draw_control.on_draw(self.handle_draw) + self.map.add(self.draw_control) # initialize data and colorbars self.geojson = None self.tooltip = None From 0444799a66c265d5ec288d9e9d07422cc741ab8d Mon Sep 17 00:00:00 2001 From: tsutterley Date: Tue, 25 Jun 2024 13:37:05 -0700 Subject: [PATCH 6/6] save sliderule crs metadata as json in parquet output --- clients/python/sliderule/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/python/sliderule/io.py b/clients/python/sliderule/io.py index 34b92c90e..f21c51b21 100644 --- a/clients/python/sliderule/io.py +++ b/clients/python/sliderule/io.py @@ -494,7 +494,7 @@ def to_parquet(gdf, filename, **kwargs): pass # save CRS to JSON crs = pyproj.CRS.from_string(kwargs['crs']) - output['crs'] = crs.to_string() + output['crs'] = crs.to_json_dict() # save each region following GeoJSON specification output['type'] = 'FeatureCollection' output['features'] = []