From af09818bd0bc98b51b90bb680eb840e74502aa3a Mon Sep 17 00:00:00 2001 From: PennyHow Date: Tue, 4 Jun 2024 11:53:57 -0100 Subject: [PATCH] revert --- .github/workflows/process_test.yml | 2 +- .gitignore | 1 - setup.py | 3 +- src/pypromice/process/L1toL2.py | 67 ++------- src/pypromice/process/L2toL3.py | 40 +++++- src/pypromice/process/aws.py | 109 +++++---------- src/pypromice/process/get_l2.py | 46 ------- src/pypromice/process/get_l3.py | 128 +++++------------- .../process/{join_l2.py => join_l3.py} | 36 ++--- 9 files changed, 129 insertions(+), 303 deletions(-) delete mode 100644 src/pypromice/process/get_l2.py mode change 100644 => 100755 src/pypromice/process/get_l3.py rename src/pypromice/process/{join_l2.py => join_l3.py} (83%) diff --git a/.github/workflows/process_test.yml b/.github/workflows/process_test.yml index e7180f6f..5ba3d51a 100644 --- a/.github/workflows/process_test.yml +++ b/.github/workflows/process_test.yml @@ -38,7 +38,7 @@ jobs: run: | mkdir $GITHUB_WORKSPACE/out/ for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do - python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2.py -v $GITHUB_WORKSPACE/main/src/pypromice/process/variables.csv -m $GITHUB_WORKSPACE/main/src/pypromice/process/metadata.csv -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/ + python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l3.py -v $GITHUB_WORKSPACE/main/src/pypromice/process/variables.csv -m $GITHUB_WORKSPACE/main/src/pypromice/process/metadata.csv -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/ done - name: Upload test output uses: actions/upload-artifact@v3 diff --git a/.gitignore b/.gitignore index cebe88d2..065beda1 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,3 @@ src/pypromice/postprocess/positions.csv # sqlite db files *.db -*.bak diff --git a/setup.py b/setup.py index 1755a7a3..2a4638d3 100644 --- a/setup.py +++ b/setup.py @@ -42,9 +42,8 @@ 'console_scripts': [ 'get_promice_data = pypromice.get.get_promice_data:get_promice_data', 'get_l0tx = pypromice.tx.get_l0tx:get_l0tx', - 'get_l2 = pypromice.process.get_l2:get_l2', - 'join_l2 = pypromice.process.join_l2:join_l2', 'get_l3 = pypromice.process.get_l3:get_l3', + 'join_l3 = pypromice.process.join_l3:join_l3', 'get_watsontx = pypromice.tx.get_watsontx:get_watsontx', 'get_bufr = pypromice.postprocess.get_bufr:main', 'get_msg = pypromice.tx.get_msg:get_msg' diff --git a/src/pypromice/process/L1toL2.py b/src/pypromice/process/L1toL2.py index 69bf5c48..73ca1a7d 100644 --- a/src/pypromice/process/L1toL2.py +++ b/src/pypromice/process/L1toL2.py @@ -30,18 +30,7 @@ def toL2( eps_clear=9.36508e-6, emissivity=0.97, ) -> xr.Dataset: - '''Process one Level 1 (L1) product to Level 2. - In this step we do: - - manual flagging and adjustments - - automated QC: persistence, percentile - - custom filter: gps_alt filter, NaN t_rad removed from dlr & ulr - - smoothing of tilt and rot - - calculation of rh with regards to ice in subfreezin conditions - - caluclation of cloud coverage - - correction of dsr and usr for tilt - - filtering of dsr based on a theoritical TOA irradiance and grazing light - - calculation of albedo - - caluclation of directional wind speed + '''Process one Level 1 (L1) product to Level 2 Parameters ---------- @@ -96,20 +85,10 @@ def toL2( ds['dlr'] = ds.dlr.where(ds.t_rad.notnull()) ds['ulr'] = ds.ulr.where(ds.t_rad.notnull()) - # calculating realtive humidity with regard to ice T_100 = _getTempK(T_0) ds['rh_u_cor'] = correctHumidity(ds['rh_u'], ds['t_u'], T_0, T_100, ews, ei0) - if ds.attrs['number_of_booms']==2: - ds['rh_l_cor'] = correctHumidity(ds['rh_l'], ds['t_l'], - T_0, T_100, ews, ei0) - - if hasattr(ds,'t_i'): - if ~ds['t_i'].isnull().all(): - ds['rh_i_cor'] = correctHumidity(ds['rh_i'], ds['t_i'], - T_0, T_100, ews, ei0) - # Determiune cloud cover for on-ice stations cc = calcCloudCoverage(ds['t_u'], T_0, eps_overcast, eps_clear, # Calculate cloud coverage ds['dlr'], ds.attrs['station_id']) @@ -197,52 +176,22 @@ def toL2( ds['precip_u_cor'], ds['precip_u_rate'] = correctPrecip(ds['precip_u'], ds['wspd_u']) if ds.attrs['number_of_booms']==2: + ds['rh_l_cor'] = correctHumidity(ds['rh_l'], ds['t_l'], # Correct relative humidity + T_0, T_100, ews, ei0) + if ~ds['precip_l'].isnull().all() and precip_flag: # Correct precipitation ds['precip_l_cor'], ds['precip_l_rate']= correctPrecip(ds['precip_l'], ds['wspd_l']) - # Get directional wind speed - ds['wdir_u'] = ds['wdir_u'].where(ds['wspd_u'] != 0) - ds['wspd_x_u'], ds['wspd_y_u'] = calcDirWindSpeeds(ds['wspd_u'], ds['wdir_u']) - - if ds.attrs['number_of_booms']==2: - ds['wdir_l'] = ds['wdir_l'].where(ds['wspd_l'] != 0) - ds['wspd_x_l'], ds['wspd_y_l'] = calcDirWindSpeeds(ds['wspd_l'], ds['wdir_l']) - - if hasattr(ds, 'wdir_i'): - if ~ds['wdir_i'].isnull().all() and ~ds['wspd_i'].isnull().all(): - ds['wdir_i'] = ds['wdir_i'].where(ds['wspd_i'] != 0) - ds['wspd_x_i'], ds['wspd_y_i'] = calcDirWindSpeeds(ds['wspd_i'], ds['wdir_i']) - + if hasattr(ds,'t_i'): + if ~ds['t_i'].isnull().all(): # Instantaneous msg processing + ds['rh_i_cor'] = correctHumidity(ds['rh_i'], ds['t_i'], # Correct relative humidity + T_0, T_100, ews, ei0) ds = clip_values(ds, vars_df) return ds -def calcDirWindSpeeds(wspd, wdir, deg2rad=np.pi/180): - '''Calculate directional wind speed from wind speed and direction - - Parameters - ---------- - wspd : xr.Dataarray - Wind speed data array - wdir : xr.Dataarray - Wind direction data array - deg2rad : float - Degree to radians coefficient. The default is np.pi/180 - - Returns - ------- - wspd_x : xr.Dataarray - Wind speed in X direction - wspd_y : xr.Datarray - Wind speed in Y direction - ''' - wspd_x = wspd * np.sin(wdir * deg2rad) - wspd_y = wspd * np.cos(wdir * deg2rad) - return wspd_x, wspd_y - - def calcCloudCoverage(T, T_0, eps_overcast, eps_clear, dlr, station_id): '''Calculate cloud cover from T and T_0 diff --git a/src/pypromice/process/L2toL3.py b/src/pypromice/process/L2toL3.py index 9751122e..a71a4028 100755 --- a/src/pypromice/process/L2toL3.py +++ b/src/pypromice/process/L2toL3.py @@ -7,10 +7,7 @@ def toL3(L2, T_0=273.15, z_0=0.001, R_d=287.05, eps=0.622, es_0=6.1071, es_100=1013.246): - '''Process one Level 2 (L2) product to Level 3 (L3) meaning calculating all - derived variables: - - Sensible fluxes - + '''Process one Level 2 (L2) product to Level 3 (L3) Parameters ---------- @@ -35,6 +32,9 @@ def toL3(L2, T_0=273.15, z_0=0.001, R_d=287.05, eps=0.622, es_0=6.1071, T_100 = _getTempK(T_0) # Get steam point temperature as K + ds['wdir_u'] = ds['wdir_u'].where(ds['wspd_u'] != 0) # Get directional wind speed + ds['wspd_x_u'], ds['wspd_y_u'] = calcDirWindSpeeds(ds['wspd_u'], ds['wdir_u']) + # Upper boom bulk calculation T_h_u = ds['t_u'].copy() # Copy for processing p_h_u = ds['p_u'].copy() @@ -85,9 +85,41 @@ def toL3(L2, T_0=273.15, z_0=0.001, R_d=287.05, eps=0.622, es_0=6.1071, q_h_l = cleanSpHumid(q_h_l, T_h_l, Tsurf_h, p_h_l, RH_cor_h_l) # Clean sp.humid values ds['qh_l'] = (('time'), q_h_l.data) + ds['wdir_l'] = ds['wdir_l'].where(ds['wspd_l'] != 0) # Get directional wind speed + ds['wspd_x_l'], ds['wspd_y_l'] = calcDirWindSpeeds(ds['wspd_l'], ds['wdir_l']) + + if hasattr(ds, 'wdir_i'): + if ~ds['wdir_i'].isnull().all() and ~ds['wspd_i'].isnull().all(): # Instantaneous msg processing + ds['wdir_i'] = ds['wdir_i'].where(ds['wspd_i'] != 0) # Get directional wind speed + ds['wspd_x_i'], ds['wspd_y_i'] = calcDirWindSpeeds(ds['wspd_i'], ds['wdir_i']) + return ds +def calcDirWindSpeeds(wspd, wdir, deg2rad=np.pi/180): + '''Calculate directional wind speed from wind speed and direction + + Parameters + ---------- + wspd : xr.Dataarray + Wind speed data array + wdir : xr.Dataarray + Wind direction data array + deg2rad : float + Degree to radians coefficient. The default is np.pi/180 + + Returns + ------- + wspd_x : xr.Dataarray + Wind speed in X direction + wspd_y : xr.Datarray + Wind speed in Y direction + ''' + wspd_x = wspd * np.sin(wdir * deg2rad) + wspd_y = wspd * np.cos(wdir * deg2rad) + return wspd_x, wspd_y + + def calcHeatFlux(T_0, T_h, Tsurf_h, rho_atm, WS_h, z_WS, z_T, nu, q_h, p_h, kappa=0.4, WS_lim=1., z_0=0.001, g=9.82, es_0=6.1071, eps=0.622, gamma=16., L_sub=2.83e6, L_dif_max=0.01, c_pd=1005., aa=0.7, diff --git a/src/pypromice/process/aws.py b/src/pypromice/process/aws.py index 2ce64de2..58440595 100644 --- a/src/pypromice/process/aws.py +++ b/src/pypromice/process/aws.py @@ -91,30 +91,12 @@ def getL1(self): logger.info('Level 1 processing...') self.L0 = [addBasicMeta(item, self.vars) for item in self.L0] self.L1 = [toL1(item, self.vars) for item in self.L0] - - if self.merge_flag: - self.L1A = self.hard_merge(self.L1) - else: - self.L1A = reduce(xr.Dataset.combine_first, self.L1) + self.L1A = reduce(xr.Dataset.combine_first, self.L1) def getL2(self): '''Perform L1 to L2 data processing''' logger.info('Level 2 processing...') self.L2 = toL2(self.L1A, vars_df=self.vars) - self.L2 = self.resample(self.L2) - self.L2 = reformat_time(self.L2) - - # Switch gps_lon to negative (degrees_east) - # Do this here, and NOT in addMeta, otherwise we switch back to positive - # when calling getMeta in joinL2! PJW - if self.L2.attrs['station_id'] not in ['UWN', 'Roof_GEUS', 'Roof_PROMICE']: - self.L2['gps_lon'] = self.L2['gps_lon'] * -1 - - # Add variable attributes and metadata - self.L2 = self.addAttributes(self.L2) - - # Round all values to specified decimals places - self.L2 = roundValues(self.L2, self.vars) def getL3(self): '''Perform L2 to L3 data processing, including resampling and metadata @@ -122,48 +104,31 @@ def getL3(self): logger.info('Level 3 processing...') self.L3 = toL3(self.L2) - def resample(self, dataset): - '''Resample dataset to specific temporal resolution (based on input - data type)''' + # Resample L3 product f = [l.attrs['format'] for l in self.L0] if 'raw' in f or 'STM' in f: logger.info('Resampling to 10 minute') - resampled = resampleL2(dataset, '10min') + self.L3 = resampleL3(self.L3, '10min') else: - resampled = resampleL2(dataset, '60min') + self.L3 = resampleL3(self.L3, '60min') logger.info('Resampling to hour') - return resampled - - def merge_flag(self): - '''Determine if hard merging is needed, based on whether a hard - merge_type flag is defined in any of the configs''' - f = [l.attrs['merge_type'] for l in self.L0] - if 'hard' in f: - return True - else: - return False - - def hard_merge(self, dataset_list): - '''Determine positions where hard merging should occur, combine - data and append to list of combined data chunks, then hard merge all - combined data chunks. This should be called in instances where there - needs to be a clear break between input datasets, such as when a station - is moved (and we do not want the GPS position jumping)''' - # Define positions where hard merging should occur - m=[] - f = [l.attrs['merge_type'] for l in self.L0] - [m.append(i) for i, item in enumerate(f) if item=='hard'] - - # Perform combine between hard merge breaks and append to list of combined data - combined=[] - for i in range(len(m[:-1])): - combined.append(reduce(xr.Dataset.combine_first, dataset_list[m[i]:m[i+1]])) - combined.append(reduce(xr.Dataset.combine_first, dataset_list[m[-1]:])) - - # Hard merge all combined datasets together - return reduce(xr.Dataset.update, combined) - - + + # Re-format time + t = self.L3['time'].values + self.L3['time'] = list(t) + + # Switch gps_lon to negative (degrees_east) + # Do this here, and NOT in addMeta, otherwise we switch back to positive + # when calling getMeta in joinL3! PJW + if self.L3.attrs['station_id'] not in ['UWN', 'Roof_GEUS', 'Roof_PROMICE']: + self.L3['gps_lon'] = self.L3['gps_lon'] * -1 + + # Add variable attributes and metadata + self.L3 = self.addAttributes(self.L3) + + # Round all values to specified decimals places + self.L3 = roundValues(self.L3, self.vars) + def addAttributes(self, L3): '''Add variable and attribute metadata @@ -400,12 +365,6 @@ def getL0(infile, nodata, cols, skiprows, file_version, ds = xr.Dataset.from_dataframe(df) return ds -def reformat_time(dataset): - '''Re-format time''' - t = dataset['time'].values - dataset['time'] = list(t) - return dataset - def addBasicMeta(ds, vars_df): ''' Use a variable lookup table DataFrame to add the basic metadata to the xarray dataset. This is later amended to finalise L3 @@ -753,8 +712,8 @@ def getMeta(m_file=None, delimiter=','): pass return meta -def resampleL2(ds_h, t): - '''Resample L2 AWS data, e.g. hourly to daily average. This uses pandas +def resampleL3(ds_h, t): + '''Resample L3 AWS data, e.g. hourly to daily average. This uses pandas DataFrame resampling at the moment as a work-around to the xarray Dataset resampling. As stated, xarray resampling is a lengthy process that takes ~2-3 minutes per operation: ds_d = ds_h.resample({'time':"1D"}).mean() @@ -922,7 +881,7 @@ def testAddAll(self): self.assertTrue(d.attrs['station_id']=='TEST') self.assertIsInstance(d.attrs['references'], str) - def testL0toL2(self): + def testL0toL3(self): '''Test L0 to L3 processing''' try: import pypromice @@ -931,23 +890,19 @@ def testL0toL2(self): except: pAWS = AWS('../test/test_config1.toml', '../test/') pAWS.process() - self.assertIsInstance(pAWS.L2, xr.Dataset) - self.assertTrue(pAWS.L2.attrs['station_id']=='TEST1') - - def testCLIgetl2(self): - '''Test get_l2 CLI''' - exit_status = os.system('get_l2 -h') - self.assertEqual(exit_status, 0) + self.assertIsInstance(pAWS.L3, xr.Dataset) + self.assertTrue(pAWS.L3.attrs['station_id']=='TEST1') - def testCLIjoinl2(self): - '''Test join_l2 CLI''' - exit_status = os.system('join_l2 -h') - self.assertEqual(exit_status, 0) - def testCLIgetl3(self): '''Test get_l3 CLI''' exit_status = os.system('get_l3 -h') self.assertEqual(exit_status, 0) + + def testCLIjoinl3(self): + '''Test join_l3 CLI''' + exit_status = os.system('join_l3 -h') + self.assertEqual(exit_status, 0) + #------------------------------------------------------------------------------ if __name__ == "__main__": diff --git a/src/pypromice/process/get_l2.py b/src/pypromice/process/get_l2.py deleted file mode 100644 index 7e28be7c..00000000 --- a/src/pypromice/process/get_l2.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -import logging, os, sys, unittest -from argparse import ArgumentParser -from pypromice.process.aws import AWS - -def parse_arguments_l2(): - parser = ArgumentParser(description="AWS L2 processor") - - parser.add_argument('-c', '--config_file', type=str, required=True, - help='Path to config (TOML) file') - parser.add_argument('-i', '--inpath', default='data', type=str, required=True, - help='Path to input data') - parser.add_argument('-o', '--outpath', default=None, type=str, required=False, - help='Path where to write output') - parser.add_argument('-v', '--variables', default=None, type=str, - required=False, help='File path to variables look-up table') - parser.add_argument('-m', '--metadata', default=None, type=str, - required=False, help='File path to metadata') - args = parser.parse_args() - return args - -def get_l2(): - args = parse_arguments_l2() - - logging.basicConfig( - format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", - level=logging.INFO, - stream=sys.stdout, - ) - - station_name = args.config_file.split('/')[-1].split('.')[0] - station_path = os.path.join(args.inpath, station_name) - - if os.path.exists(station_path): - aws = AWS(args.config_file, station_path, args.variables, args.metadata) - else: - aws = AWS(args.config_file, args.inpath, args.variables, args.metadata) - - aws.process() - - if args.outpath is not None: - aws.write(args.outpath) - -if __name__ == "__main__": - get_l2() - diff --git a/src/pypromice/process/get_l3.py b/src/pypromice/process/get_l3.py old mode 100644 new mode 100755 index ac1cbad8..4cc18436 --- a/src/pypromice/process/get_l3.py +++ b/src/pypromice/process/get_l3.py @@ -1,108 +1,46 @@ #!/usr/bin/env python -import os, unittest, pkg_resources -import pandas as pd -import numpy as np -import xarray as xr +import logging, os, sys, unittest from argparse import ArgumentParser -from pypromice.process import getVars, getMeta, addMeta, getColNames, \ - roundValues, resampleL2, writeAll -from pypromice.process.L1toL2 import correctPrecip -from pypromice.process.L2toL3 import toL3 -from sys import exit +from pypromice.process.aws import AWS -def parse_arguments_getl3(debug_args=None): - parser = ArgumentParser(description="AWS L3 script for the processing L3 data from L2 and merging the L3 data with its historical site. An hourly, daily and monthly L3 data product is outputted to the defined output path") - parser.add_argument('-s', '--file1', type=str, required=True, nargs='+', - help='Path to source L2 file') - # here will come additional arguments for the merging with historical stations - parser.add_argument('-v', '--variables', default=None, type=str, required=False, - help='Path to variables look-up table .csv file for variable name retained'''), - parser.add_argument('-m', '--metadata', default=None, type=str, required=False, - help='Path to metadata table .csv file for metadata information'''), - parser.add_argument('-d', '--datatype', default='raw', type=str, required=False, - help='Data type to output, raw or tx') - args = parser.parse_args(args=debug_args) - args.file1 = ' '.join(args.file1) - args.folder_gcnet = ' '.join(args.folder_gcnet) - args.folder_promice = ' '.join(args.folder_promice) +def parse_arguments_l3(): + parser = ArgumentParser(description="AWS L3 processor") + + parser.add_argument('-c', '--config_file', type=str, required=True, + help='Path to config (TOML) file') + parser.add_argument('-i', '--inpath', default='data', type=str, required=True, + help='Path to input data') + parser.add_argument('-o', '--outpath', default=None, type=str, required=False, + help='Path where to write output') + parser.add_argument('-v', '--variables', default=None, type=str, + required=False, help='File path to variables look-up table') + parser.add_argument('-m', '--metadata', default=None, type=str, + required=False, help='File path to metadata') + args = parser.parse_args() return args +def get_l3(): + args = parse_arguments_l3() -def loadArr(infile): - if infile.split('.')[-1].lower() in 'csv': - df = pd.read_csv(infile) - df['time'] = pd.to_datetime(df['time']).dt.tz_localize(None) - df = df.set_index('time') - ds = xr.Dataset.from_dataframe(df) + logging.basicConfig( + format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", + level=logging.INFO, + stream=sys.stdout, + ) - elif infile.split('.')[-1].lower() in 'nc': - ds = xr.open_dataset(infile) - - try: - name = ds.attrs['station_name'] - except: - name = infile.split('/')[-1].split('.')[0].split('_hour')[0].split('_10min')[0] - - print(f'{name} array loaded from {infile}') - return ds, name + station_name = args.config_file.split('/')[-1].split('.')[0] + station_path = os.path.join(args.inpath, station_name) -def get_l3(): - args = parse_arguments_getl3() - - # Check files - if os.path.isfile(args.file1): - # Load L2 data arrays - ds1, n1 = loadArr(args.file1) - - # converts to L3: - # - derives sensible heat fluxes - # - more to come - ds1 = toL3(ds1) - - # here will come the merging with historical data + if os.path.exists(station_path): + aws = AWS(args.config_file, station_path, args.variables, args.metadata) else: - print(f'Invalid file {args.file1}') - exit() + aws = AWS(args.config_file, args.inpath, args.variables, args.metadata) - # Get hourly, daily and monthly datasets - print('Resampling L3 data to hourly, daily and monthly resolutions...') - l3_h = resampleL2(ds1, '60min') - l3_d = resampleL2(ds1, '1D') - l3_m = resampleL2(ds1, 'M') - - print(f'Adding variable information from {args.variables}...') - - # Load variables look-up table - var = getVars(args.variables) - - # Round all values to specified decimals places - l3_h = roundValues(l3_h, var) - l3_d = roundValues(l3_d, var) - l3_m = roundValues(l3_m, var) + aws.process() + + if args.outpath is not None: + aws.write(args.outpath) - # Get columns to keep - if hasattr(ds1, 'p_l'): - col_names = getColNames(var, 2, args.datatype.lower()) - else: - col_names = getColNames(var, 1, args.datatype.lower()) - - # Assign station id - for l in [l3_h, l3_d, l3_m]: - l.attrs['station_id'] = n1 - - # Assign metadata - print(f'Adding metadata from {args.metadata}...') - m = getMeta(args.metadata) - l3_h = addMeta(l3_h, m) - l3_d = addMeta(l3_d, m) - l3_m = addMeta(l3_m, m) - - # Set up output path - out = os.path.join(args.outpath, site_id) - - # Write to files - writeAll(out, site_id, l3_h, l3_d, l3_m, col_names) - print(f'Files saved to {os.path.join(out, site_id)}...') -# %% if __name__ == "__main__": get_l3() + diff --git a/src/pypromice/process/join_l2.py b/src/pypromice/process/join_l3.py similarity index 83% rename from src/pypromice/process/join_l2.py rename to src/pypromice/process/join_l3.py index 943d9afa..2e0d3fe3 100644 --- a/src/pypromice/process/join_l2.py +++ b/src/pypromice/process/join_l3.py @@ -4,15 +4,15 @@ import xarray as xr from argparse import ArgumentParser from pypromice.process import getVars, getMeta, addMeta, getColNames, \ - roundValues, resampleL2, writeAll + roundValues, resampleL3, writeAll from pypromice.process.L1toL2 import correctPrecip def parse_arguments_join(): - parser = ArgumentParser(description="AWS L2 joiner for merging together two L2 products, for example an L2 RAW and L2 TX data product. An hourly, daily and monthly L2 data product is outputted to the defined output path") + parser = ArgumentParser(description="AWS L3 joiner for merging together two L3 products, for example an L3 RAW and L3 TX data product. An hourly, daily and monthly L3 data product is outputted to the defined output path") parser.add_argument('-s', '--file1', type=str, required=True, - help='Path to source L2 file, which will be preferenced in merge process') + help='Path to source L3 file, which will be preferenced in merge process') parser.add_argument('-t', '--file2', type=str, required=True, - help='Path to target L2 file, which will be used to fill gaps in merge process') + help='Path to target L3 file, which will be used to fill gaps in merge process') parser.add_argument('-o', '--outpath', default=os.getcwd(), type=str, required=True, help='Path where to write output') parser.add_argument('-v', '--variables', default=None, type=str, required=False, @@ -41,7 +41,7 @@ def loadArr(infile): return ds, name -def join_l2(): +def join_l3(): args = parse_arguments_join() # Check files @@ -87,10 +87,10 @@ def join_l2(): exit() # Get hourly, daily and monthly datasets - print('Resampling L2 data to hourly, daily and monthly resolutions...') - l2_h = resampleL2(all_ds, '60min') - l2_d = resampleL2(all_ds, '1D') - l2_m = resampleL2(all_ds, 'M') + print('Resampling L3 data to hourly, daily and monthly resolutions...') + l3_h = resampleL3(all_ds, '60min') + l3_d = resampleL3(all_ds, '1D') + l3_m = resampleL3(all_ds, 'M') print(f'Adding variable information from {args.variables}...') @@ -98,9 +98,9 @@ def join_l2(): var = getVars(args.variables) # Round all values to specified decimals places - l2_h = roundValues(l2_h, var) - l2_d = roundValues(l2_d, var) - l2_m = roundValues(l2_m, var) + l3_h = roundValues(l3_h, var) + l3_d = roundValues(l3_d, var) + l3_m = roundValues(l3_m, var) # Get columns to keep if hasattr(all_ds, 'p_l'): @@ -109,22 +109,22 @@ def join_l2(): col_names = getColNames(var, 1, args.datatype.lower()) # Assign station id - for l in [l2_h, l2_d, l2_m]: + for l in [l3_h, l3_d, l3_m]: l.attrs['station_id'] = name # Assign metadata print(f'Adding metadata from {args.metadata}...') m = getMeta(args.metadata) - l2_h = addMeta(l2_h, m) - l2_d = addMeta(l2_d, m) - l2_m = addMeta(l2_m, m) + l3_h = addMeta(l3_h, m) + l3_d = addMeta(l3_d, m) + l3_m = addMeta(l3_m, m) # Set up output path out = os.path.join(args.outpath, name) # Write to files - writeAll(out, name, l2_h, l2_d, l2_m, col_names) + writeAll(out, name, l3_h, l3_d, l3_m, col_names) print(f'Files saved to {os.path.join(out, name)}...') if __name__ == "__main__": - join_l2() + join_l3()