From aeefc29846a807cb9373c6dd85c7c63859198638 Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Tue, 3 Sep 2024 10:31:03 +0200 Subject: [PATCH 1/5] adding GlacioBasis folder to inputs and appending --- src/pypromice/process/aws.py | 3 +- src/pypromice/process/join_l3.py | 50 ++++++++++++++++--- src/pypromice/resources/__init__.py | 1 + .../variable_aliases_GlacioBasis.csv | 8 +++ 4 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 src/pypromice/resources/variable_aliases_GlacioBasis.csv diff --git a/src/pypromice/process/aws.py b/src/pypromice/process/aws.py index 1d203ad4..899cccb0 100644 --- a/src/pypromice/process/aws.py +++ b/src/pypromice/process/aws.py @@ -55,7 +55,8 @@ def __init__( """ assert os.path.isfile(config_file), "cannot find " + config_file assert os.path.isdir(inpath), "cannot find " + inpath - logger.info( + + logger.debug( "AWS(" f"config_file={config_file}," f" inpath={inpath}," diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index b550a8c1..8bf41d6a 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -49,6 +49,13 @@ def parse_arguments_joinl3(debug_args=None): required=False, help="Path to GC-Net historical L1 folder", ) + parser.add_argument( + "-gb", + "--folder_glaciobasis", + type=str, + required=False, + help="Path to GlacioBasis historical data folder", + ) parser.add_argument( "-o", @@ -147,7 +154,7 @@ def readNead(infile): ds = df.to_xarray() ds.attrs = meta - # renaming variables + # renaming variables if GC-Net file_path = pypromice.resources.DEFAULT_VARIABLES_ALIASES_GCNET_PATH var_name = pd.read_csv(file_path) var_name = var_name.set_index("old_name").GEUS_name @@ -160,7 +167,7 @@ def readNead(infile): # renaming variables to the GEUS names ds = ds.rename(var_name) - + # variables always dropped from the historical GC-Net files # could be move to the config files at some point standard_vars_to_drop = [ @@ -201,6 +208,28 @@ def loadArr(infile, isNead): df = pd.read_csv(infile) df["time"] = pd.to_datetime(df["time"]).dt.tz_localize(None) df = df.set_index("time") + + # renaming variables if GlacioBasis + file_path = pypromice.resources.DEFAULT_VARIABLES_ALIASES_GLACIOBASIS_PATH + var_name = pd.read_csv(file_path) + var_name = var_name.set_index("old_name").GEUS_name + msk = [v for v in var_name.index if v in df.columns] + var_name = var_name.loc[msk].to_dict() + + # postprocessing for glaciobasis + if 'ice_ablation' in df.columns: + df['ice_ablation'] = (-df['ice_ablation'].diff()).cumsum() + df['z_surf_combined'] = df['ice_ablation'].values + + # renaming variables to the GEUS names + df = df.rename(columns=var_name) + + # variables always dropped from the historical GC-Net files + # could be move to the config files at some point + standard_vars_to_drop = ["I"] + + # Drop the variables if they are present in the dataset + df = df.drop(columns=[var for var in standard_vars_to_drop if var in df.columns]) ds = xr.Dataset.from_dataframe(df) elif infile.split(".")[-1].lower() in "nc": @@ -366,7 +395,7 @@ def build_station_list(config_folder: str, target_station_site: str) -> list: return station_info_list -def join_l3(config_folder, site, folder_l3, folder_gcnet, outpath, variables, metadata): +def join_l3(config_folder, site, folder_l3, folder_gcnet, folder_glaciobasis, outpath, variables, metadata): # Get the list of station information dictionaries associated with the given site list_station_info = build_station_list(config_folder, site) @@ -377,9 +406,17 @@ def join_l3(config_folder, site, folder_l3, folder_gcnet, outpath, variables, me filepath = os.path.join(folder_l3, stid, stid + "_hour.nc") isNead = False - if station_info["project"].lower() in ["historical gc-net"]: - filepath = os.path.join(folder_gcnet, stid + ".csv") - isNead = True + + if not os.path.isfile(filepath): + if station_info["project"].lower() in ["historical gc-net"]: + filepath = os.path.join(folder_gcnet, stid + ".csv") + isNead = True + + if station_info["project"].lower() in ["glaciobasis"]: + filepath = os.path.join(folder_glaciobasis, stid + ".csv") + isNead = False + + # import pdb; pdb.set_trace() if not os.path.isfile(filepath): logger.error( "\n***\n" @@ -551,6 +588,7 @@ def main(): args.site, args.folder_l3, args.folder_gcnet, + args.folder_glaciobasis, args.outpath, args.variables, args.metadata, diff --git a/src/pypromice/resources/__init__.py b/src/pypromice/resources/__init__.py index 4308f2ae..b9ac3ff5 100644 --- a/src/pypromice/resources/__init__.py +++ b/src/pypromice/resources/__init__.py @@ -7,6 +7,7 @@ DEFAULT_METADATA_PATH = (Path(__file__).parent / "file_attributes.csv").absolute() DEFAULT_VARIABLES_PATH = (Path(__file__).parent / "variables.csv").absolute() DEFAULT_VARIABLES_ALIASES_GCNET_PATH = (Path(__file__).parent / "variable_aliases_GC-Net.csv").absolute() +DEFAULT_VARIABLES_ALIASES_GLACIOBASIS_PATH = (Path(__file__).parent / "variable_aliases_GlacioBasis.csv").absolute() def load_metadata(path: Union[None, str, Path] = None) -> Dict[str, str]: """ diff --git a/src/pypromice/resources/variable_aliases_GlacioBasis.csv b/src/pypromice/resources/variable_aliases_GlacioBasis.csv new file mode 100644 index 00000000..a7935ecb --- /dev/null +++ b/src/pypromice/resources/variable_aliases_GlacioBasis.csv @@ -0,0 +1,8 @@ +GEUS_name,old_name +wspd_u,wspd +rh_u_cor,rh_u_corr +dsr_cor,dsr_corr +usr_cor,usr_corr +cc,cloud_cover +z_boom_u,z_boom +z_ice_surf,ice_ablation From b6f7b70e0fcacae375ce2302d56d2c926f178677 Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Tue, 3 Sep 2024 14:36:55 +0200 Subject: [PATCH 2/5] fixing test --- tests/e2e/test_process.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/test_process.py b/tests/e2e/test_process.py index 2fd2d16c..043e9ee0 100644 --- a/tests/e2e/test_process.py +++ b/tests/e2e/test_process.py @@ -164,6 +164,7 @@ def test_full_e2e(self): site=site_id, folder_l3=output_l3.as_posix(), folder_gcnet=folder_gc_net.as_posix(), + folder_glaciobasis='.', outpath=output_site_l3.as_posix(), variables=None, metadata=None, From a795018f0abe280d553ee8658df941ab339aaf9c Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Tue, 3 Sep 2024 10:31:03 +0200 Subject: [PATCH 3/5] adding GlacioBasis folder to inputs and appending --- src/pypromice/process/aws.py | 3 +- src/pypromice/process/join_l3.py | 50 ++++++++++++++++--- src/pypromice/resources/__init__.py | 1 + .../variable_aliases_GlacioBasis.csv | 8 +++ 4 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 src/pypromice/resources/variable_aliases_GlacioBasis.csv diff --git a/src/pypromice/process/aws.py b/src/pypromice/process/aws.py index 1d203ad4..899cccb0 100644 --- a/src/pypromice/process/aws.py +++ b/src/pypromice/process/aws.py @@ -55,7 +55,8 @@ def __init__( """ assert os.path.isfile(config_file), "cannot find " + config_file assert os.path.isdir(inpath), "cannot find " + inpath - logger.info( + + logger.debug( "AWS(" f"config_file={config_file}," f" inpath={inpath}," diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index b550a8c1..8bf41d6a 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -49,6 +49,13 @@ def parse_arguments_joinl3(debug_args=None): required=False, help="Path to GC-Net historical L1 folder", ) + parser.add_argument( + "-gb", + "--folder_glaciobasis", + type=str, + required=False, + help="Path to GlacioBasis historical data folder", + ) parser.add_argument( "-o", @@ -147,7 +154,7 @@ def readNead(infile): ds = df.to_xarray() ds.attrs = meta - # renaming variables + # renaming variables if GC-Net file_path = pypromice.resources.DEFAULT_VARIABLES_ALIASES_GCNET_PATH var_name = pd.read_csv(file_path) var_name = var_name.set_index("old_name").GEUS_name @@ -160,7 +167,7 @@ def readNead(infile): # renaming variables to the GEUS names ds = ds.rename(var_name) - + # variables always dropped from the historical GC-Net files # could be move to the config files at some point standard_vars_to_drop = [ @@ -201,6 +208,28 @@ def loadArr(infile, isNead): df = pd.read_csv(infile) df["time"] = pd.to_datetime(df["time"]).dt.tz_localize(None) df = df.set_index("time") + + # renaming variables if GlacioBasis + file_path = pypromice.resources.DEFAULT_VARIABLES_ALIASES_GLACIOBASIS_PATH + var_name = pd.read_csv(file_path) + var_name = var_name.set_index("old_name").GEUS_name + msk = [v for v in var_name.index if v in df.columns] + var_name = var_name.loc[msk].to_dict() + + # postprocessing for glaciobasis + if 'ice_ablation' in df.columns: + df['ice_ablation'] = (-df['ice_ablation'].diff()).cumsum() + df['z_surf_combined'] = df['ice_ablation'].values + + # renaming variables to the GEUS names + df = df.rename(columns=var_name) + + # variables always dropped from the historical GC-Net files + # could be move to the config files at some point + standard_vars_to_drop = ["I"] + + # Drop the variables if they are present in the dataset + df = df.drop(columns=[var for var in standard_vars_to_drop if var in df.columns]) ds = xr.Dataset.from_dataframe(df) elif infile.split(".")[-1].lower() in "nc": @@ -366,7 +395,7 @@ def build_station_list(config_folder: str, target_station_site: str) -> list: return station_info_list -def join_l3(config_folder, site, folder_l3, folder_gcnet, outpath, variables, metadata): +def join_l3(config_folder, site, folder_l3, folder_gcnet, folder_glaciobasis, outpath, variables, metadata): # Get the list of station information dictionaries associated with the given site list_station_info = build_station_list(config_folder, site) @@ -377,9 +406,17 @@ def join_l3(config_folder, site, folder_l3, folder_gcnet, outpath, variables, me filepath = os.path.join(folder_l3, stid, stid + "_hour.nc") isNead = False - if station_info["project"].lower() in ["historical gc-net"]: - filepath = os.path.join(folder_gcnet, stid + ".csv") - isNead = True + + if not os.path.isfile(filepath): + if station_info["project"].lower() in ["historical gc-net"]: + filepath = os.path.join(folder_gcnet, stid + ".csv") + isNead = True + + if station_info["project"].lower() in ["glaciobasis"]: + filepath = os.path.join(folder_glaciobasis, stid + ".csv") + isNead = False + + # import pdb; pdb.set_trace() if not os.path.isfile(filepath): logger.error( "\n***\n" @@ -551,6 +588,7 @@ def main(): args.site, args.folder_l3, args.folder_gcnet, + args.folder_glaciobasis, args.outpath, args.variables, args.metadata, diff --git a/src/pypromice/resources/__init__.py b/src/pypromice/resources/__init__.py index 4308f2ae..b9ac3ff5 100644 --- a/src/pypromice/resources/__init__.py +++ b/src/pypromice/resources/__init__.py @@ -7,6 +7,7 @@ DEFAULT_METADATA_PATH = (Path(__file__).parent / "file_attributes.csv").absolute() DEFAULT_VARIABLES_PATH = (Path(__file__).parent / "variables.csv").absolute() DEFAULT_VARIABLES_ALIASES_GCNET_PATH = (Path(__file__).parent / "variable_aliases_GC-Net.csv").absolute() +DEFAULT_VARIABLES_ALIASES_GLACIOBASIS_PATH = (Path(__file__).parent / "variable_aliases_GlacioBasis.csv").absolute() def load_metadata(path: Union[None, str, Path] = None) -> Dict[str, str]: """ diff --git a/src/pypromice/resources/variable_aliases_GlacioBasis.csv b/src/pypromice/resources/variable_aliases_GlacioBasis.csv new file mode 100644 index 00000000..a7935ecb --- /dev/null +++ b/src/pypromice/resources/variable_aliases_GlacioBasis.csv @@ -0,0 +1,8 @@ +GEUS_name,old_name +wspd_u,wspd +rh_u_cor,rh_u_corr +dsr_cor,dsr_corr +usr_cor,usr_corr +cc,cloud_cover +z_boom_u,z_boom +z_ice_surf,ice_ablation From 7bdf2c981d5d3262f2a6d4acc0ce2426817f74e7 Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Tue, 3 Sep 2024 14:36:55 +0200 Subject: [PATCH 4/5] fixing test --- tests/e2e/test_process.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/test_process.py b/tests/e2e/test_process.py index 2fd2d16c..043e9ee0 100644 --- a/tests/e2e/test_process.py +++ b/tests/e2e/test_process.py @@ -164,6 +164,7 @@ def test_full_e2e(self): site=site_id, folder_l3=output_l3.as_posix(), folder_gcnet=folder_gc_net.as_posix(), + folder_glaciobasis='.', outpath=output_site_l3.as_posix(), variables=None, metadata=None, From d519924ed6c33c0dc98e941a93b0954e6038dfbe Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Fri, 8 Nov 2024 13:29:45 +0100 Subject: [PATCH 5/5] Update variable_aliases_GlacioBasis.csv --- src/pypromice/resources/variable_aliases_GlacioBasis.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pypromice/resources/variable_aliases_GlacioBasis.csv b/src/pypromice/resources/variable_aliases_GlacioBasis.csv index a7935ecb..18ee9150 100644 --- a/src/pypromice/resources/variable_aliases_GlacioBasis.csv +++ b/src/pypromice/resources/variable_aliases_GlacioBasis.csv @@ -1,6 +1,6 @@ GEUS_name,old_name wspd_u,wspd -rh_u_cor,rh_u_corr +rh_u_wrt_ice_or_water,rh_u_corr dsr_cor,dsr_corr usr_cor,usr_corr cc,cloud_cover