From e96c9434a596c9ddd67b13f3095acc431ce9992c Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Tue, 16 Jan 2024 17:45:30 -0500 Subject: [PATCH 1/4] fix: step 4 cleanups --- .../analysis_db/A02c_IOWAGA_thredds_prior.py | 4 +- .../analysis_db/B03_plot_spectra_ov.py | 6 +-- src/icesat2_tracks/config/IceSAT2_startup.py | 47 +++---------------- 3 files changed, 10 insertions(+), 47 deletions(-) diff --git a/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py b/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py index 7b49becd..3a7175f2 100644 --- a/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py +++ b/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py @@ -1,4 +1,4 @@ -import os, sys +import sys from icesat2_tracks.config.IceSAT2_startup import ( mconfig, @@ -425,8 +425,6 @@ def test_nan_frac(imask): except: target_name = "A02_" + track_name + "_hindcast_fail" - -# %% def plot_prior(Prior, axx): angle = Prior["incident_angle"][ "value" diff --git a/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py b/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py index e69f6728..77ef2706 100644 --- a/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py +++ b/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py @@ -25,15 +25,15 @@ track_name, batch_key, test_flag = io.init_from_input( - sys.argv + sys.argv # TODO: Handle via CLI ) # loads standard experiment hemis, batch = batch_key.split("_") load_path = mconfig["paths"]["work"] + batch_key + "/B02_spectra/" -load_file = load_path + "B02_" + track_name # + '.nc' +load_file = load_path + "B02_" + track_name plot_path = ( mconfig["paths"]["plot"] + "/" + hemis + "/" + batch_key + "/" + track_name + "/" -) +) # TODO: Update with pathlib MT.mkdirs_r(plot_path) Gk = xr.open_dataset(load_file + "_gFT_k.nc") diff --git a/src/icesat2_tracks/config/IceSAT2_startup.py b/src/icesat2_tracks/config/IceSAT2_startup.py index b14cc5b4..1de693e2 100644 --- a/src/icesat2_tracks/config/IceSAT2_startup.py +++ b/src/icesat2_tracks/config/IceSAT2_startup.py @@ -1,27 +1,10 @@ import os import pathlib -### -# THIS FILE IS A LOCAL FILE -# it is not maintained via git, it contains configs specific to the machine -### - -#os.environ["DISPLAY"] = "localhost:10.0" -# 14, 16, work -#standart libraries: -import numpy as np -import matplotlib import matplotlib.pyplot as plt - -import matplotlib.colors as colors -import pandas as pd from icesat2_tracks.local_modules import m_colormanager_ph3 as M_color from icesat2_tracks.local_modules import m_tools_ph3 as MT -from icesat2_tracks.local_modules import m_general_ph3 as M - import string -import xarray as xr - ## Read folders and configuration paths config_dir_path = os.path.dirname(__file__) mconfig=MT.json_load('config',config_dir_path) @@ -37,12 +20,10 @@ #load colorscheme color_schemes=M_color.color(path=mconfig['paths']['config'], name='color_def') - lstrings =iter([i+') ' for i in list(string.ascii_lowercase)]) # define journal fig sizes fig_sizes = mconfig['fig_sizes']['AMS'] - SMALL_SIZE = 8 MEDIUM_SIZE = 10 BIGGER_SIZE = 12 @@ -52,11 +33,11 @@ plt.rc('font', size=SMALL_SIZE, serif='Helvetica Neue', weight='normal') # controls default text sizes plt.rc('text', usetex='false') plt.rc('axes', titlesize=MEDIUM_SIZE, labelweight='normal') # fontsize of the axes title -plt.rc('axes', labelsize=SMALL_SIZE, labelweight='normal') #, family='bold') # fontsize of the x and y labels +plt.rc('axes', labelsize=SMALL_SIZE, labelweight='normal') # fontsize of the x and y labels plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels plt.rc('legend', fontsize=SMALL_SIZE, frameon=False) # legend fontsize -plt.rc('figure', titlesize=MEDIUM_SIZE, titleweight='bold', autolayout=True) #, family='bold') # fontsize of the figure title +plt.rc('figure', titlesize=MEDIUM_SIZE, titleweight='bold', autolayout=True) # fontsize of the figure title plt.rc('path', simplify=True) plt.rcParams['figure.figsize'] = (10, 8) plt.rcParams['pcolor.shading'] = 'auto' @@ -66,15 +47,7 @@ plt.rc('axes', labelsize= MEDIUM_SIZE, labelweight='normal') plt.rc('axes.spines', top= False, right=False ) - - -def font_for_print(): - - SMALL_SIZE = 6 - MEDIUM_SIZE = 8 - BIGGER_SIZE = 10 # not used. CP - legend_properties = {'weight':'bold'} # not used. CP - +def font_for_print(SMALL_SIZE = 6, MEDIUM_SIZE = 8): plt.rc('font', size=SMALL_SIZE, serif='Helvetica Neue', weight='normal') # controls default text sizes plt.rc('text', usetex='false') plt.rc('axes', titlesize=MEDIUM_SIZE, labelweight='normal') # fontsize of the axes title @@ -85,21 +58,13 @@ def font_for_print(): plt.rc('figure', titlesize=MEDIUM_SIZE, titleweight='bold', autolayout=True) #, family='bold') # fontsize of the figure title plt.rc('axes', labelsize= SMALL_SIZE, labelweight='normal') -def font_for_pres(): - - SMALL_SIZE = 10 - MEDIUM_SIZE = 12 - BIGGER_SIZE = 14 # not used. CP - legend_properties = {'weight':'bold'} # not used. CP - - - plt.rc('font', size=SMALL_SIZE, serif='Helvetica Neue', weight='normal') # controls default text sizes +def font_for_pres(SMALL_SIZE = 10, MEDIUM_SIZE = 12): + plt.rc('font', size=SMALL_SIZE, serif='Helvetica Neue', weight='normal') # controls default text sizes plt.rc('text', usetex='false') plt.rc('axes', titlesize=MEDIUM_SIZE, labelweight='normal') # fontsize of the axes title plt.rc('axes', labelsize=SMALL_SIZE, labelweight='normal') #, family='bold') # fontsize of the x and y labels plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels plt.rc('legend', fontsize=SMALL_SIZE, frameon=False) # legend fontsize - plt.rc('figure', titlesize=MEDIUM_SIZE, titleweight='bold', autolayout=True) #, family='bold') # fontsize of the figure title - + plt.rc('figure', titlesize=MEDIUM_SIZE, titleweight='bold', autolayout=True) # fontsize of the figure title plt.rc('axes', labelsize= SMALL_SIZE, labelweight='normal') From 3ccb7c04abf99f1902f5efbff39457c545a29bea Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Tue, 16 Jan 2024 19:36:35 -0500 Subject: [PATCH 2/4] fix: add explicit import xarray --- .../analysis_db/B01_SL_load_single_file.py | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py b/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py index 37fbfc8d..84fa58d0 100644 --- a/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py +++ b/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py @@ -1,24 +1,21 @@ -# """ This file open a ICEsat2 tbeam_stats.pyrack applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3.11 """ -import os, sys +import sys +import datetime +import copy +import imp + +import xarray as xr +from sliderule import sliderule, icesat2 + from icesat2_tracks.config.IceSAT2_startup import ( mconfig, - xr, color_schemes, font_for_pres, plt, ) - -import geopandas as gpd - -from sliderule import sliderule, icesat2, earthdata - -import shapely -from ipyleaflet import basemaps, Map, GeoData - import icesat2_tracks.ICEsat2_SI_tools.sliderule_converter_tools as sct import icesat2_tracks.ICEsat2_SI_tools.io as io import icesat2_tracks.ICEsat2_SI_tools.beam_stats as beam_stats @@ -26,10 +23,6 @@ from icesat2_tracks.local_modules import m_general_ph3 as M -import h5py, imp, copy -import datetime - - xr.set_options(display_style="text") @@ -39,7 +32,6 @@ sys.argv ) # loads standard experiment -# 20190502052058_05180312_005_01 plot_flag = True hemis = batch_key.split("_")[0] @@ -118,8 +110,6 @@ # main routine for defining the x coordinate and sacing table data - - def make_B01_dict(table_data, split_by_beam=True, to_hdf5=False): """ converts a GeoDataFrame from Sliderule to GeoDataFrames for each beam witht the correct columns and names @@ -181,7 +171,6 @@ def make_B01_dict(table_data, split_by_beam=True, to_hdf5=False): Ti[kk]["heights_c_weighted_mean"] = Ti[kk]["h_mean"].copy() Ti[kk]["heights_c_std"] = Ti[kk]["h_sigma"].copy() - segment = track_name.split("_")[1][-2:] ID_name = sct.create_ID_name(gdf.iloc[0], segment=segment) print(ID_name) From 309b36fa80aca21055acbd43a144fa7a1deb8365 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Tue, 16 Jan 2024 20:58:37 -0500 Subject: [PATCH 3/4] fix: remove use of the imp module Also, organize imports in scripts --- .../analysis_db/B01_SL_load_single_file.py | 4 +- .../analysis_db/B02_make_spectra_gFT.py | 43 ++++++++----------- .../analysis_db/B03_plot_spectra_ov.py | 21 ++------- 3 files changed, 24 insertions(+), 44 deletions(-) diff --git a/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py b/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py index 84fa58d0..0f04f155 100644 --- a/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py +++ b/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py @@ -5,7 +5,6 @@ import sys import datetime import copy -import imp import xarray as xr from sliderule import sliderule, icesat2 @@ -156,7 +155,7 @@ def make_B01_dict(table_data, split_by_beam=True, to_hdf5=False): # define reference point and then define 'x' table_data = copy.copy(gdf) -imp.reload(sct) + # the reference point is defined as the most equatorward point of the polygon. # It's distance from the equator is subtracted from the distance of each photon. table_data = sct.define_x_coordinate_from_data(table_data) @@ -192,7 +191,6 @@ def make_B01_dict(table_data, split_by_beam=True, to_hdf5=False): F_atl06.save_light(path=plot_path, name="B01b_ATL06_corrected.png") plt.close() -imp.reload(beam_stats) if plot_flag: font_for_pres() F = M.figure_axis_xy(8, 4.3, view_scale=0.6) diff --git a/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py b/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py index 28536c01..b23dcfea 100644 --- a/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py +++ b/src/icesat2_tracks/analysis_db/B02_make_spectra_gFT.py @@ -1,31 +1,27 @@ -import sys - - """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -from icesat2_tracks.config.IceSAT2_startup import mconfig, xr, plt, np - +import copy +import datetime +import h5py +import time +import sys -from threadpoolctl import threadpool_info, threadpool_limits +import numpy as np +import xarray as xr from pprint import pprint +from scipy.ndimage.measurements import label +from threadpoolctl import threadpool_info, threadpool_limits - -import h5py +import icesat2_tracks.ICEsat2_SI_tools.generalized_FT as gFT import icesat2_tracks.ICEsat2_SI_tools.io as io import icesat2_tracks.ICEsat2_SI_tools.spectral_estimates as spec - -import time -import imp -import copy -from icesat2_tracks.local_modules.m_spectrum_ph3 import spicke_remover -import datetime -import icesat2_tracks.ICEsat2_SI_tools.generalized_FT as gFT -from scipy.ndimage.measurements import label +import icesat2_tracks.local_modules.m_general_ph3 as M +import icesat2_tracks.local_modules.m_spectrum_ph3 as spicke_remover import icesat2_tracks.local_modules.m_tools_ph3 as MT -from icesat2_tracks.local_modules import m_general_ph3 as M +from icesat2_tracks.config.IceSAT2_startup import mconfig, plt import tracemalloc @@ -99,7 +95,7 @@ def linear_gap_fill(F, key_lead, key_int): Ib = Gd[group[1]] ratio = Ia["x"][:].size / Ib["x"][:].size if (ratio > 10) | (ratio < 0.1): - print("bad data ratio ", ratio, 1 / ratio) + # print("bad data ratio ", ratio, 1 / ratio) # TODO: add logger bad_ratio_flag = True if (np.array(nan_fraction).mean() > 0.95) | bad_ratio_flag: @@ -118,7 +114,6 @@ def linear_gap_fill(F, key_lead, key_int): exit() # test LS with an even grid where missing values are set to 0 -imp.reload(spec) print(Gd.keys()) Gi = Gd[list(Gd.keys())[0]] # to select a test beam dist = io.get_beam_var_hdf_store(Gd[list(Gd.keys())[0]], "dist") @@ -153,10 +148,10 @@ def linear_gap_fill(F, key_lead, key_int): print("define global xlims") dist_list = np.array([np.nan, np.nan]) for k in all_beams: - print(k) + # print(k) # TODO: add logger hkey = "heights_c_weighted_mean" x = Gd[k + "/dist"][:] - print(x[0], x[-1]) + # print(x[0], x[-1]) # TODO: add logger dist_list = np.vstack([dist_list, [x[0], x[-1]]]) xlims = np.nanmin(dist_list[:, 0]) - dx, np.nanmin(dist_list[:, 1]) @@ -172,7 +167,7 @@ def linear_gap_fill(F, key_lead, key_int): for k in all_beams: dist_i = io.get_beam_var_hdf_store(Gd[k], "dist") x_mask = (dist_i > xlims[0]) & (dist_i < xlims[1]) - print(k, sum(x_mask["dist"]) / (xlims[1] - xlims[0])) + # print(k, sum(x_mask["dist"]) / (xlims[1] - xlims[0])) # TODO: add logger print("-reduced frequency resolution") @@ -199,7 +194,7 @@ def linear_gap_fill(F, key_lead, key_int): Gi = io.get_beam_hdf_store(Gd[k]) x_mask = (Gi["dist"] > xlims[0]) & (Gi["dist"] < xlims[1]) if sum(x_mask) / (xlims[1] - xlims[0]) < 0.005: - print("------------------- not data in beam found; skip") + # print("------------------- not data in beam found; skip") # TODO: add logger continue Gd_cut = Gi[x_mask] @@ -219,7 +214,7 @@ def linear_gap_fill(F, key_lead, key_int): # compute slope spectra !! dd = np.gradient(dd) - dd, _ = spicke_remover(dd, spreed=10, verbose=False) + dd, _ = spicke_remover.spicke_remover(dd, spreed=10, verbose=False) dd_nans = (np.isnan(dd)) + (Gd_cut["N_photos"] <= 5) # using gappy data diff --git a/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py b/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py index 77ef2706..0d9a97bf 100644 --- a/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py +++ b/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py @@ -3,26 +3,15 @@ This is python 3 """ import sys - -from icesat2_tracks.config.IceSAT2_startup import ( - mconfig, - xr, - color_schemes, - plt, - np, - font_for_print, -) - -import icesat2_tracks.ICEsat2_SI_tools.io as io -import icesat2_tracks.ICEsat2_SI_tools.spectral_estimates as spec - import time +import numpy as np +import xarray as xr from matplotlib.gridspec import GridSpec +import icesat2_tracks.ICEsat2_SI_tools.io as io import icesat2_tracks.ICEsat2_SI_tools.generalized_FT as gFT -from scipy.ndimage.measurements import label import icesat2_tracks.local_modules.m_tools_ph3 as MT from icesat2_tracks.local_modules import m_general_ph3 as M - +from icesat2_tracks.config.IceSAT2_startup import mconfig, color_schemes, plt, font_for_print track_name, batch_key, test_flag = io.init_from_input( sys.argv # TODO: Handle via CLI @@ -40,8 +29,6 @@ Gx = xr.open_dataset(load_file + "_gFT_x.nc") Gfft = xr.open_dataset(load_file + "_FFT.nc") -time.sleep(2) - all_beams = mconfig["beams"]["all_beams"] high_beams = mconfig["beams"]["high_beams"] From f6e1580dbe9d84a4c819c9a64795b99f9c2a3ad6 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Tue, 16 Jan 2024 21:04:12 -0500 Subject: [PATCH 4/4] fix: organize imports, remove unused --- .../analysis_db/A02c_IOWAGA_thredds_prior.py | 27 ++++++++----------- .../analysis_db/B03_plot_spectra_ov.py | 1 - src/icesat2_tracks/config/IceSAT2_startup.py | 4 ++- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py b/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py index 3a7175f2..2da5d760 100644 --- a/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py +++ b/src/icesat2_tracks/analysis_db/A02c_IOWAGA_thredds_prior.py @@ -1,26 +1,21 @@ import sys - -from icesat2_tracks.config.IceSAT2_startup import ( - mconfig, - xr, - color_schemes, - plt, - font_for_print, - np, -) +import datetime import h5py +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.font_manager as font_for_print +from matplotlib.gridspec import GridSpec +import xarray as xr +from siphon.catalog import TDSCatalog + +from icesat2_tracks.config.IceSAT2_startup import mconfig import icesat2_tracks.ICEsat2_SI_tools.io as io import icesat2_tracks.ICEsat2_SI_tools.wave_tools as waves import icesat2_tracks.local_modules.m_tools_ph3 as MT import icesat2_tracks.local_modules.m_general_ph3 as M - -from siphon.catalog import TDSCatalog - -import pandas as pd - -import datetime -from matplotlib.gridspec import GridSpec +from icesat2_tracks.config.IceSAT2_startup import color_schemes color_schemes.colormaps2(21) diff --git a/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py b/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py index 0d9a97bf..ca7984d2 100644 --- a/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py +++ b/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py @@ -3,7 +3,6 @@ This is python 3 """ import sys -import time import numpy as np import xarray as xr from matplotlib.gridspec import GridSpec diff --git a/src/icesat2_tracks/config/IceSAT2_startup.py b/src/icesat2_tracks/config/IceSAT2_startup.py index 1de693e2..d1e27938 100644 --- a/src/icesat2_tracks/config/IceSAT2_startup.py +++ b/src/icesat2_tracks/config/IceSAT2_startup.py @@ -1,9 +1,11 @@ import os import pathlib +import string + import matplotlib.pyplot as plt + from icesat2_tracks.local_modules import m_colormanager_ph3 as M_color from icesat2_tracks.local_modules import m_tools_ph3 as MT -import string ## Read folders and configuration paths config_dir_path = os.path.dirname(__file__)