Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: logging 0 - add logger configuration and update logging in step 1 #138

Merged
merged 43 commits into from
Mar 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
74c7b63
remove suppress STDOUT call
hollandjg Feb 29, 2024
c27a0ad
replace print and echo calls with logger info calls
hollandjg Feb 29, 2024
32f28fd
feat: add basi . logging support to b01
hollandjg Feb 29, 2024
6c732ae
feat: add basic logging to m_general_ph3
hollandjg Feb 29, 2024
36a24c8
feat: remove unneeded logging commands / arguments
hollandjg Feb 29, 2024
29a0399
feat: aadd basic logging to b01 when run as __main__
hollandjg Feb 29, 2024
ecb45b8
feat: add logging and foncifto app
hollandjg Feb 29, 2024
9ad7d83
feat: add logging and debugs to colormanager
hollandjg Feb 29, 2024
648dd5b
refactor: remove duplicated code
hollandjg Feb 29, 2024
fbb7fe8
fix: string in debugger
hollandjg Feb 29, 2024
c9dcd88
revert changes to run_job calls
hollandjg Feb 29, 2024
8847762
refactor: move main app callback to top
hollandjg Feb 29, 2024
44a87eb
Update src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py
hollandjg Feb 29, 2024
5523bd0
fix: change default logging level for step 1 when run from the script…
hollandjg Mar 1, 2024
58be37b
change to using an enum for log levels.
hollandjg Mar 4, 2024
5fa9c66
doc: add more documentation to the log level
hollandjg Mar 4, 2024
54cc478
fix: add invoke_without_command=True
hollandjg Mar 4, 2024
0522dd1
fix: remove invoke without command
hollandjg Mar 4, 2024
9746c36
Merge branch 'main' into feat-add-logger
hollandjg Mar 4, 2024
0fb9eb6
chore: remove unused import
hollandjg Mar 4, 2024
16c3fc9
chore: refactor log level mapping
hollandjg Mar 4, 2024
194dce1
fix: remove unused parameter from correct_separate
hollandjg Mar 4, 2024
73ebb24
test: remove verbose flag from tests
hollandjg Mar 4, 2024
6e84db9
Update src/icesat2_tracks/app.py
hollandjg Mar 4, 2024
cb57489
Merge branch 'main' into feat-add-logger
hollandjg Mar 4, 2024
8e91491
Update src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py
hollandjg Mar 5, 2024
aaab734
Update src/icesat2_tracks/local_modules/m_colormanager_ph3.py
hollandjg Mar 5, 2024
f9779d5
Update src/icesat2_tracks/local_modules/m_colormanager_ph3.py
hollandjg Mar 5, 2024
a044c61
Update src/icesat2_tracks/local_modules/m_colormanager_ph3.py
hollandjg Mar 5, 2024
4cc2da5
Update src/icesat2_tracks/local_modules/m_colormanager_ph3.py
hollandjg Mar 5, 2024
cd12cd5
Update src/icesat2_tracks/local_modules/m_colormanager_ph3.py
hollandjg Mar 5, 2024
3cc72fe
Update src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py
hollandjg Mar 5, 2024
fd26f77
Update src/icesat2_tracks/local_modules/m_colormanager_ph3.py
hollandjg Mar 5, 2024
6ebff19
Update src/icesat2_tracks/local_modules/m_colormanager_ph3.py
hollandjg Mar 5, 2024
e988514
Merge branch 'main' into feat-add-logger
hollandjg Mar 5, 2024
a849457
Apply suggestions from code review
hollandjg Mar 5, 2024
39b145f
Update src/icesat2_tracks/local_modules/m_colormanager_ph3.py
hollandjg Mar 5, 2024
96f4239
Merge branch 'main' into feat-add-logger
hollandjg Mar 5, 2024
529bf31
Merge branch 'main' into feat-add-logger
hollandjg Mar 7, 2024
b22e178
Merge branch 'main' into feat-add-logger
hollandjg Mar 11, 2024
b4350ad
Merge branch 'main' into feat-add-logger
hollandjg Mar 11, 2024
077c1b5
Merge branch 'main' into feat-add-logger
hollandjg Mar 13, 2024
db8ea2b
fix: some broken logger calls
hollandjg Mar 13, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
292 changes: 147 additions & 145 deletions src/icesat2waves/analysis_db/B01_SL_load_single_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import copy
from pathlib import Path
import warnings
import logging

import xarray as xr
from sliderule import icesat2
Expand Down Expand Up @@ -40,6 +41,8 @@
makeapp,
)

_logger = logging.getLogger(__name__)


def make_B01_dict(table_data, split_by_beam=True, to_hdf5=False):
"""
Expand Down Expand Up @@ -91,7 +94,6 @@ def run_B01_SL_load_single_file(
ID_flag: bool = True,
plot_flag: bool = True,
output_dir: str = typer.Option(..., callback=validate_output_dir),
verbose: bool = False,
):
"""
Open an ICEsat2 tbeam_stats.pyrack, apply filters and corrections, and output smoothed photon heights on a regular grid in an .nc file.
Expand All @@ -109,174 +111,174 @@ def run_B01_SL_load_single_file(
matplotlib.use("Agg") # prevent plot windows from opening

# Select region and retrieve batch of tracks
with suppress_stdout(verbose):
track_name, batch_key, ID_flag = io.init_from_input(
[
None,
track_name,
batch_key,
ID_flag,
] # init_from_input expects sys.argv with 4 elements
) # loads standard experiment

hemis = batch_key.split("_")[0]
track_name, batch_key, ID_flag = io.init_from_input(
[
None,
track_name,
batch_key,
ID_flag,
] # init_from_input expects sys.argv with 4 elements
) # loads standard experiment

hemis = batch_key.split("_")[0]

workdir, plotsdir = update_paths_mconfig(output_dir, mconfig)

save_path = Path(workdir, batch_key, "B01_regrid")
save_path.mkdir(parents=True, exist_ok=True)

save_path_json = Path(workdir, batch_key, "A01b_ID")
save_path_json.mkdir(parents=True, exist_ok=True)

ATL03_track_name = f"ATL03_{track_name}.h5"

# Configure SL Session
icesat2.init("slideruleearth.io")

# plot the ground tracks in geographic location
# Generate ATL06-type segments using the ATL03-native photon classification
# Use the ocean classification for photons with a confidence parameter to 2 or higher (low confidence or better)

# YAPC alternative
params_yapc = {
"srt": 1,
"len": 20,
"ats": 3,
"res": 10,
"dist_in_seg": False, # if False units of len and res are in meters
"track": 0,
"pass_invalid": False,
"cnf": 2,
"cnt": 20,
"sigma_r_max": 4, # maximum standard deviation of photon in extend
"maxi": 10,
"yapc": dict(
knn=0, win_h=6, win_x=11, min_ph=4, score=100
), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff
# "yapc": dict(knn=0, win_h=3, win_x=11, min_ph=4, score=50), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff
"atl03_geo_fields": ["dem_h"],
}

workdir, plotsdir = update_paths_mconfig(output_dir, mconfig)
maximum_height = 30 # (meters) maximum height past dem_h correction
_logger.info("Fetching ATL03 data from sliderule")
gdf = icesat2.atl06p(params_yapc, resources=[ATL03_track_name])
_logger.info("Finished fetching ATL03 data from sliderule")
gdf = sct.correct_and_remove_height(gdf, maximum_height)

save_path = Path(workdir, batch_key, "B01_regrid")
save_path.mkdir(parents=True, exist_ok=True)
cdict = dict()
for s, b in zip(
gdf["spot"].unique(), ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"]
):
cdict[s] = color_schemes.rels[b]

save_path_json = Path(workdir, batch_key, "A01b_ID")
save_path_json.mkdir(parents=True, exist_ok=True)
font_for_pres()
F_atl06 = M.FigureAxisXY(6.5, 5, view_scale=0.6)
F_atl06.fig.suptitle(track_name)

ATL03_track_name = "ATL03_" + track_name + ".h5"
beam_stats.plot_ATL06_track_data(gdf, cdict)

# Configure SL Session
icesat2.init("slideruleearth.io")
# main routine for defining the x coordinate and sacing table data

# plot the ground tracks in geographic location
# Generate ATL06-type segments using the ATL03-native photon classification
# Use the ocean classification for photons with a confidence parameter to 2 or higher (low confidence or better)

# YAPC alternative
params_yapc = {
"srt": 1,
"len": 20,
"ats": 3,
"res": 10,
"dist_in_seg": False, # if False units of len and res are in meters
"track": 0,
"pass_invalid": False,
"cnf": 2,
"cnt": 20,
"sigma_r_max": 4, # maximum standard deviation of photon in extend
"maxi": 10,
"yapc": dict(
knn=0, win_h=6, win_x=11, min_ph=4, score=100
), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff
# "yapc": dict(knn=0, win_h=3, win_x=11, min_ph=4, score=50), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff
"atl03_geo_fields": ["dem_h"],
}

maximum_height = 30 # (meters) maximum height past dem_h correction
print("STARTS")
print("Fetching ATL03 data from sliderule")
gdf = icesat2.atl06p(params_yapc, resources=[ATL03_track_name])
print("ENDS")
gdf = sct.correct_and_remove_height(gdf, maximum_height)

cdict = dict()
for s, b in zip(
gdf["spot"].unique(), ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"]
):
cdict[s] = color_schemes.rels[b]
# define reference point and then define 'x'
table_data = copy.copy(gdf)

font_for_pres()
F_atl06 = M.FigureAxisXY(6.5, 5, view_scale=0.6)
F_atl06.fig.suptitle(track_name)
# the reference point is defined as the most equatorward point of the polygon.
# It's distance from the equator is subtracted from the distance of each photon.
table_data = sct.define_x_coordinate_from_data(table_data)
table_time = table_data["time"]
table_data.drop(columns=["time"], inplace=True)

# renames columns and splits beams
Ti = make_B01_dict(table_data, split_by_beam=True, to_hdf5=True)

with warnings.catch_warnings():
warnings.simplefilter(
"ignore", category=SettingWithCopyWarning
) # TODO: remove when warnings are handled
for kk in Ti.keys():
Ti[kk]["dist"] = Ti[kk]["x"].copy()
Ti[kk]["heights_c_weighted_mean"] = Ti[kk]["h_mean"].copy()
Ti[kk]["heights_c_std"] = Ti[kk]["h_sigma"].copy()

segment = track_name.split("_")[1][-2:]
ID_name = sct.create_ID_name(gdf.iloc[0], segment=segment)
echoparam("ID_name", ID_name)
io.write_track_to_HDF5(Ti, ID_name + "_B01_binned", save_path) # regridding heights

beam_stats.plot_ATL06_track_data(gdf, cdict)
# plot the ground tracks in geographic location

# main routine for defining the x coordinate and sacing table data
all_beams = mconfig["beams"]["all_beams"]
high_beams = mconfig["beams"]["high_beams"]
low_beams = mconfig["beams"]["low_beams"]

# define reference point and then define 'x'
table_data = copy.copy(gdf)
D = beam_stats.derive_beam_statistics(Ti, all_beams, Lmeter=12.5e3, dx=10)

# the reference point is defined as the most equatorward point of the polygon.
# It's distance from the equator is subtracted from the distance of each photon.
table_data = sct.define_x_coordinate_from_data(table_data)
table_time = table_data["time"]
table_data.drop(columns=["time"], inplace=True)
# save figure from above:
plot_path = Path(plotsdir, hemis, batch_key, ID_name)
plot_path.mkdir(parents=True, exist_ok=True)

# renames columns and splits beams
Ti = make_B01_dict(table_data, split_by_beam=True, to_hdf5=True)
F_atl06.save_light(path=plot_path, name="B01b_ATL06_corrected.png")
plt.close()

with warnings.catch_warnings():
warnings.simplefilter(
"ignore", category=SettingWithCopyWarning
) # TODO: remove when warnings are handled
for kk in Ti.keys():
Ti[kk]["dist"] = Ti[kk]["x"].copy()
Ti[kk]["heights_c_weighted_mean"] = Ti[kk]["h_mean"].copy()
Ti[kk]["heights_c_std"] = Ti[kk]["h_sigma"].copy()
if plot_flag:
font_for_pres()
F = M.FigureAxisXY(8, 4.3, view_scale=0.6)
beam_stats.plot_beam_statistics(
D,
high_beams,
low_beams,
color_schemes.rels,
track_name=track_name
+ "| ascending ="
+ str(sct.ascending_test_distance(gdf)),
)

F.save_light(path=plot_path, name="B01b_beam_statistics.png")
plt.close()

segment = track_name.split("_")[1][-2:]
ID_name = sct.create_ID_name(gdf.iloc[0], segment=segment)
echoparam("ID_name", ID_name)
io.write_track_to_HDF5(Ti, ID_name + "_B01_binned", save_path) # regridding heights
# plot the ground tracks in geographic location
gdf[::100].plot(markersize=0.1, figsize=(4, 6))
plt.title(
track_name + "\nascending =" + str(sct.ascending_test_distance(gdf)),
loc="left",
)
M.save_anyfig(plt.gcf(), path=plot_path, name="B01_track.png")
plt.close()

# plot the ground tracks in geographic location
_logger.info("write A01b .json")
DD = {"case_ID": ID_name, "tracks": {}}

all_beams = mconfig["beams"]["all_beams"]
high_beams = mconfig["beams"]["high_beams"]
low_beams = mconfig["beams"]["low_beams"]
DD["tracks"]["ATL03"] = f"ATL10-{track_name}"

D = beam_stats.derive_beam_statistics(Ti, all_beams, Lmeter=12.5e3, dx=10)
start_pos = abs(table_data.lats).argmin()
end_pos = abs(table_data.lats).argmax()

# save figure from above:
plot_path = Path(plotsdir, hemis, batch_key, ID_name)
plot_path.mkdir(parents=True, exist_ok=True)
# add other pars:
DD["pars"] = {
"poleward": sct.ascending_test(gdf),
"region": "0",
"start": {
"longitude": table_data.lons[start_pos],
"latitude": table_data.lats[start_pos],
"seg_dist_x": float(table_data.x[start_pos]),
"delta_time": datetime.datetime.timestamp(table_time[start_pos]),
},
"end": {
"longitude": table_data.lons[end_pos],
"latitude": table_data.lats[end_pos],
"seg_dist_x": float(table_data.x[end_pos]),
"delta_time": datetime.datetime.timestamp(table_time[end_pos]),
},
}

F_atl06.save_light(path=plot_path, name="B01b_ATL06_corrected.png")
plt.close()
MT.json_save2(name="A01b_ID_" + ID_name, path=save_path_json, data=DD)

if plot_flag:
font_for_pres()
F = M.FigureAxisXY(8, 4.3, view_scale=0.6)
beam_stats.plot_beam_statistics(
D,
high_beams,
low_beams,
color_schemes.rels,
track_name=track_name
+ "| ascending ="
+ str(sct.ascending_test_distance(gdf)),
)

F.save_light(path=plot_path, name="B01b_beam_statistics.png")
plt.close()

# plot the ground tracks in geographic location
gdf[::100].plot(markersize=0.1, figsize=(4, 6))
plt.title(
track_name + "\nascending =" + str(sct.ascending_test_distance(gdf)),
loc="left",
)
M.save_anyfig(plt.gcf(), path=plot_path, name="B01_track.png")
plt.close()

echo("write A01b .json")
DD = {"case_ID": ID_name, "tracks": {}}

DD["tracks"]["ATL03"] = "ATL10-" + track_name

start_pos = abs(table_data.lats).argmin()
end_pos = abs(table_data.lats).argmax()

# add other pars:
DD["pars"] = {
"poleward": sct.ascending_test(gdf),
"region": "0",
"start": {
"longitude": table_data.lons[start_pos],
"latitude": table_data.lats[start_pos],
"seg_dist_x": float(table_data.x[start_pos]),
"delta_time": datetime.datetime.timestamp(table_time[start_pos]),
},
"end": {
"longitude": table_data.lons[end_pos],
"latitude": table_data.lats[end_pos],
"seg_dist_x": float(table_data.x[end_pos]),
"delta_time": datetime.datetime.timestamp(table_time[end_pos]),
},
}

MT.json_save2(name="A01b_ID_" + ID_name, path=save_path_json, data=DD)

echo("done")
_logger.info("done")


load_file_app = makeapp(run_B01_SL_load_single_file, name="load-file")

if __name__ == "__main__":
logging.basicConfig(level=logging.WARNING)
load_file_app()
Loading