update rh_*_cor to rh_*_wrt_ice_or_water

src/pypromice/process/L1toL2.py

@@ -102,16 +102,16 @@ def toL2(
 
     # calculating realtive humidity with regard to ice
     T_100 = _getTempK(T_0)
-    ds['rh_u_cor'] = correctHumidity(ds['rh_u'], ds['t_u'],
+    ds['rh_u_wrt_ice_or_water'] = adjustHumidity(ds['rh_u'], ds['t_u'],
                                      T_0, T_100, ews, ei0)
 
     if ds.attrs['number_of_booms']==2:
-        ds['rh_l_cor'] = correctHumidity(ds['rh_l'], ds['t_l'],
+        ds['rh_l_wrt_ice_or_water'] = adjustHumidity(ds['rh_l'], ds['t_l'],
                                          T_0, T_100, ews, ei0)
 
     if hasattr(ds,'t_i'):
         if ~ds['t_i'].isnull().all():
-            ds['rh_i_cor'] = correctHumidity(ds['rh_i'], ds['t_i'],
+            ds['rh_i_wrt_ice_or_water'] = adjustHumidity(ds['rh_i'], ds['t_i'],
                                              T_0, T_100, ews, ei0)
 
     # Determiune cloud cover for on-ice stations
@@ -419,9 +419,12 @@ def calcTilt(tilt_x, tilt_y, deg2rad):
     return phi_sensor_rad, theta_sensor_rad
 
 
-def correctHumidity(rh, T, T_0, T_100, ews, ei0):                        #TODO figure out if T replicate is needed
-    '''Correct relative humidity using Groff & Gratch method, where values are
-    set when freezing and remain the original values when not freezing
+def adjustHumidity(rh, T, T_0, T_100, ews, ei0):                        #TODO figure out if T replicate is needed
+    '''Adjust relative humidity so that values are given with respect to
+    saturation over ice in subfreezing conditions, and with respect to 
+    saturation over water (as given by the instrument) above the melting 
+    point temperature. Saturation water vapors are calculated after 
+    Groff & Gratch method.
 
     Parameters
     ----------
@@ -440,7 +443,7 @@ def correctHumidity(rh, T, T_0, T_100, ews, ei0):                        #TODO f
 
     Returns
     -------
-    rh_cor : xarray.DataArray
+    rh_wrt_ice_or_water : xarray.DataArray
         Corrected relative humidity
     '''
     # Convert to hPa (Groff & Gratch)
@@ -458,8 +461,8 @@ def correctHumidity(rh, T, T_0, T_100, ews, ei0):                        #TODO f
     freezing = (T < 0) & (T > -100).values
 
     # Set to Groff & Gratch values when freezing, otherwise just rh
-    rh_cor = rh.where(~freezing, other = rh*(e_s_wtr / e_s_ice))
-    return rh_cor
+    rh_wrt_ice_or_water = rh.where(~freezing, other = rh*(e_s_wtr / e_s_ice))
+    return rh_wrt_ice_or_water
 
 
 def correctPrecip(precip, wspd):

src/pypromice/process/aws.py

@@ -94,7 +94,7 @@ def __init__(
         formats = {dataset.attrs["format"].lower() for dataset in self.L0}
         if "raw" in formats:
             self.format = "raw"
-        elif "STM" in formats:
+        elif "stm" in formats:
             self.format = "STM"
         elif "tx" in formats:
             self.format = "tx"

src/pypromice/process/resample.py

@@ -51,12 +51,13 @@ def resample_dataset(ds_h, t):
     # taking the 10 min data and using it as instantaneous values:
     is_10_minutes_timestamp = (ds_h.time.diff(dim='time') / np.timedelta64(1, 's') == 600)
     if (t == '60min') and is_10_minutes_timestamp.any():
-        cols_to_update = ['p_i', 't_i', 'rh_i', 'rh_i_cor', 'wspd_i', 'wdir_i','wspd_x_i','wspd_y_i']
+        cols_to_update = ['p_i', 't_i', 'rh_i', 'rh_i_wrt_ice_or_water', 'wspd_i', 'wdir_i','wspd_x_i','wspd_y_i']
+        cols_origin = ['p_u', 't_u', 'rh_u', 'rh_u_wrt_ice_or_water', 'wspd_u', 'wdir_u','wspd_x_u','wspd_y_u']
         timestamp_10min = ds_h.time.where(is_10_minutes_timestamp, drop=True).to_index()
         timestamp_round_hour = df_d.index
         timestamp_to_update = timestamp_round_hour.intersection(timestamp_10min)
 
-        for col in cols_to_update:
+        for col, col_org in zip(cols_to_update, cols_origin):
             if col not in df_d.columns:
                 df_d[col] = np.nan
             else:
@@ -67,7 +68,7 @@ def resample_dataset(ds_h, t):
                 timestamp_to_update = timestamp_to_update[missing_instantaneous]
             df_d.loc[timestamp_to_update, col] = ds_h.reindex(
                 time= timestamp_to_update
-                )[col.replace('_i','_u')].values
+                )[col_org].values
             if col == 'p_i':
                 df_d.loc[timestamp_to_update, col] = df_d.loc[timestamp_to_update, col].values-1000
 
@@ -95,8 +96,8 @@ def resample_dataset(ds_h, t):
 
                 df_d[var] = (p_vap.to_series().resample(t).mean() \
                            / es_wtr.to_series().resample(t).mean())*100
-                if var+'_cor' in df_d.keys():
-                    df_d[var+'_cor'] = (p_vap.to_series().resample(t).mean() \
+                if var+'_wrt_ice_or_water' in df_d.keys():
+                    df_d[var+'_wrt_ice_or_water'] = (p_vap.to_series().resample(t).mean() \
                                / es_cor.to_series().resample(t).mean())*100
 
     # passing each variable attribute to the ressample dataset

src/pypromice/process/value_clipping.py

@@ -11,8 +11,6 @@ def clip_values(
 ):
     """
     Clip values in dataset to defined "hi" and "lo" variables from dataframe.
-    There is a special treatment here for rh_u and rh_l variables, where values
-    are clipped and not assigned to NaN. This is for replication purposes
 
     Parameters
     ----------

src/pypromice/process/write.py

@@ -100,14 +100,14 @@ def prepare_and_write(
     elif t == 86400:
         # removing instantaneous values from daily and monthly files
         for v in col_names:
-            if ("_i" in v) and ("_i_" not in v):
+            if v in ['p_i', 't_i', 'rh_i', 'wspd_i', 'wdir_i', 'wspd_x_i', 'wspd_y_i']:
                 col_names.remove(v)
         out_csv = output_dir / f"{name}_day.csv"
         out_nc = output_dir / f"{name}_day.nc"
     else:
         # removing instantaneous values from daily and monthly files
         for v in col_names:
-            if ("_i" in v) and ("_i_" not in v):
+            if v in ['p_i', 't_i', 'rh_i', 'wspd_i', 'wdir_i', 'wspd_x_i', 'wspd_y_i']:
                 col_names.remove(v)
         out_csv = output_dir / f"{name}_month.csv"
         out_nc = output_dir / f"{name}_month.nc"

src/pypromice/resources/variable_aliases_GC-Net.csv

@@ -5,10 +5,10 @@ p_l,
 t_u,TA2
 t_l,TA1
 rh_u,RH2
-rh_u_cor,RH2_cor
+rh_u_wrt_ice_or_water,RH2_cor
 qh_u,Q2
 rh_l,RH1
-rh_l_cor,RH1_cor
+rh_l_wrt_ice_or_water,RH1_cor
 qh_l,Q1
 wspd_u,VW2
 wspd_l,VW1

src/pypromice/resources/variables.csv

@@ -6,10 +6,10 @@ p_l,air_pressure,Air pressure (lower boom),hPa,physicalMeasurement,time,FALSE,,6
 t_u,air_temperature,Air temperature (upper boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,"",all,1,1,1,4
 t_l,air_temperature,Air temperature (lower boom),degrees_C,physicalMeasurement,time,FALSE,,-80,40,"",two-boom,1,1,1,4
 rh_u,relative_humidity,Relative humidity (upper boom),%,physicalMeasurement,time,FALSE,,0,100,"",all,1,1,1,4
-rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,modelResult,time,FALSE,L2 or later,0,150,"",all,0,1,1,4
+rh_u_wrt_ice_or_water,relative_humidity_with_respect_to_ice_or_water,Relative humidity (upper boom) with respect to saturation over ice in subfreezing conditions and over water otherwise,%,modelResult,time,FALSE,L2 or later,0,150,"",all,0,1,1,4
 qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,modelResult,time,FALSE,L2 or later,0,100,"",all,0,1,1,4
 rh_l,relative_humidity,Relative humidity (lower boom),%,physicalMeasurement,time,FALSE,,0,100,"",two-boom,1,1,1,4
-rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,modelResult,time,FALSE,L2 or later,0,150,"",two-boom,0,1,1,4
+rh_l_wrt_ice_or_water,relative_humidity_with_respect_to_ice_or_water,Relative humidity (lower boom) with respect to saturation over ice in subfreezing conditions and over water otherwise,%,modelResult,time,FALSE,L2 or later,0,150,"",two-boom,0,1,1,4
 qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,modelResult,time,FALSE,L2 or later,0,100,,two-boom,0,1,1,4
 wspd_u,wind_speed,Wind speed (upper boom),m s-1,physicalMeasurement,time,FALSE,,0,100,wdir_u wspd_x_u wspd_y_u,all,1,1,1,4
 wspd_l,wind_speed,Wind speed (lower boom),m s-1,physicalMeasurement,time,FALSE,,0,100,wdir_l wspd_x_l wspd_y_l,two-boom,1,1,1,4
@@ -99,7 +99,7 @@ t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,phy
 p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,physicalMeasurement,time,TRUE,,-350,100,,all,1,1,1,4
 t_i,air_temperature,Air temperature (instantaneous),degrees_C,physicalMeasurement,time,TRUE,,-80,40,,all,1,1,1,4
 rh_i,relative_humidity,Relative humidity (instantaneous),%,physicalMeasurement,time,TRUE,,0,150,"",all,1,1,1,4
-rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,modelResult,time,TRUE,L2 or later,0,100,,all,0,1,1,4
+rh_i_wrt_ice_or_water,relative_humidity_with_respect_to_ice_or_water,Relative humidity (instantaneous) with respect to saturation over ice in subfreezing conditions and over water otherwise,%,modelResult,time,TRUE,L2 or later,0,100,,all,0,1,1,4
 wspd_i,wind_speed,Wind speed (instantaneous),m s-1,physicalMeasurement,time,TRUE,,0,100,wdir_i wspd_x_i wspd_y_i,all,1,1,1,4
 wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,physicalMeasurement,time,TRUE,,1,360,wspd_x_i wspd_y_i,all,1,1,1,4
 wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,modelResult,time,TRUE,L2 or later,-100,100,"",all,0,1,1,4

tests/unit/bufr_export/tx_l3_test1.csv

@@ -1,4 +1,4 @@
-time,p_u,p_l,t_u,t_l,rh_u,rh_u_cor,qh_u,rh_l,rh_l_cor,qh_l,wspd_u,wspd_l,wdir_u,wdir_l,wspd_x_u,wspd_y_u,wspd_x_l,wspd_y_l,dsr,dsr_cor,usr,usr_cor,albedo,dlr,ulr,cc,t_surf,dlhf_u,dlhf_l,dshf_u,dshf_l,z_boom_u,z_boom_l,precip_u,precip_u_cor,precip_u_rate,precip_l,precip_l_cor,precip_l_rate,t_i_1,t_i_2,t_i_3,t_i_4,t_i_5,t_i_6,t_i_7,t_i_8,t_i_9,t_i_10,t_i_11,tilt_x,tilt_y,rot,gps_lat,gps_lon,gps_alt,gps_time,gps_hdop,batt_v,fan_dc_u,fan_dc_l,t_rad,p_i,t_i,rh_i,rh_i_cor,wspd_i,wdir_i,wspd_x_i,wspd_y_i,msg_i,msg_lat,msg_lon
+time,p_u,p_l,t_u,t_l,rh_u,rh_u_wrt_ice_or_water,qh_u,rh_l,rh_l_wrt_ice_or_water,qh_l,wspd_u,wspd_l,wdir_u,wdir_l,wspd_x_u,wspd_y_u,wspd_x_l,wspd_y_l,dsr,dsr_cor,usr,usr_cor,albedo,dlr,ulr,cc,t_surf,dlhf_u,dlhf_l,dshf_u,dshf_l,z_boom_u,z_boom_l,precip_u,precip_u_cor,precip_u_rate,precip_l,precip_l_cor,precip_l_rate,t_i_1,t_i_2,t_i_3,t_i_4,t_i_5,t_i_6,t_i_7,t_i_8,t_i_9,t_i_10,t_i_11,tilt_x,tilt_y,rot,gps_lat,gps_lon,gps_alt,gps_time,gps_hdop,batt_v,fan_dc_u,fan_dc_l,t_rad,p_i,t_i,rh_i,rh_i_cor,wspd_i,wdir_i,wspd_x_i,wspd_y_i,msg_i,msg_lat,msg_lon
 2023-12-01 00:00:00,784.5,785.8,-16.32,-16.3,77.87,91.2846,1.0583,80.1,93.8804,1.0887,16.33,15.27,113.2,122.8,15.0095,-6.4331,12.8355,-8.2719,-1.9282,0.0,0.5033,0.0,,182.6197,241.7566,0.3205,-17.134,-1.6006,1.6393,30.3492,31.1479,4.1967,2.946,129.4,136.5924,0.0,304.0,315.0223,0.0,,-8.33,-7.41,-6.43,-5.9,-5.64,-5.51,-6.01,,,-12.46,0.6404,0.8809,218.9,66.482488,-46.29424,2135.0,21.0,0.9,12.72,35.15,11.81,-16.44,-213.9,-16.4,82.0,96.2012,15.48,122.3,13.0847,-8.2718,0.0,66.5026,-46.33518
 2023-12-01 01:00:00,784.1,785.5,-15.82,-15.92,76.17,88.8564,1.0798,78.75,91.956,1.1052,15.85,14.83,112.3,126.2,14.6646,-6.0144,11.9672,-8.7587,-2.0466,0.0,0.7549,0.0,,178.4223,242.0001,0.2567,-17.034,-0.5022,2.1859,43.1898,40.0931,4.2047,2.9472,129.4,136.5924,0.0,304.0,315.0223,0.0,,-8.33,-7.41,-6.43,-5.9,-5.64,-5.51,-6.01,,,-12.46,0.631,0.5809,225.6,66.482471,-46.29426,2124.0,10022.0,0.73,12.74,30.55,11.82,-16.06,-214.4,-16.2,79.4,92.9691,15.28,118.8,13.39,-7.3612,0.0,66.5026,-46.33518
 2023-12-01 02:00:00,784.1,785.5,-15.55,-15.6,76.88,89.4484,1.1146,79.15,92.1345,1.1408,15.55,14.57,117.1,129.6,13.8428,-7.0837,11.2264,-9.2873,-2.0272,0.0,0.6131,0.0,,182.2089,243.3796,0.2862,-16.6921,-0.5438,2.18,40.0149,38.6836,4.1981,2.9441,129.4,136.5924,0.0,304.0,315.0223,0.0,,-8.33,-7.41,-6.43,-5.9,-5.64,-5.51,-6.01,,,-12.46,0.3333,0.8761,231.1,66.482326,-46.294294,2121.0,20021.0,1.04,12.73,30.8,12.34,-15.74,-214.5,-15.6,79.0,91.9599,14.63,138.5,9.6941,-10.9572,0.0,66.50363,-46.20806

tests/unit/test_value_clippping.py

@@ -198,32 +198,32 @@ def test_circular_dependencies(self):
             check_dtype=True,
         )
 
-    def test_rh_corrected(self):
+    def test_rh_adjusted(self):
         variable_config = pd.DataFrame(
             columns=["field", "lo", "hi", "OOL"],
             data=[
-                ["rh_u", 0, 150, "rh_u_cor"],
-                ["rh_u_cor", 0, 150, ""],
+                ["rh_u", 0, 150, "rh_u_wrt_ice_or_water"],
+                ["rh_u_wrt_ice_or_water", 0, 150, ""],
             ],
         ).set_index("field")
 
         rows_input = []
         rows_expected = []
         # All values are within the expected range
-        rows_input.append(dict(rh_u=42, rh_u_cor=43))
-        rows_expected.append(dict(rh_u=42, rh_u_cor=43))
-        # rh_u is below range, but rh_u_cor is within range. Both should be flagged due to the OOL relationship
-        rows_input.append(dict(rh_u=-10, rh_u_cor=3))
-        rows_expected.append(dict(rh_u=np.nan, rh_u_cor=np.nan))
-        # rh_u is within range, but rh_u_cor is below range; rh_u_cor should be flagged
-        rows_input.append(dict(rh_u=54, rh_u_cor=-4))
-        rows_expected.append(dict(rh_u=54, rh_u_cor=np.nan))
-        # rh_u is above range, but rh_u_cor is within range. Both should be flagged due to the OOL relationship
-        rows_input.append(dict(rh_u=160, rh_u_cor=120))
-        rows_expected.append(dict(rh_u=np.nan, rh_u_cor=np.nan))
-        # rh_u is within range, but rh_u_cor is above range; rh_u_cor should be flagged
-        rows_input.append(dict(rh_u=100, rh_u_cor=255))
-        rows_expected.append(dict(rh_u=100, rh_u_cor=np.nan))
+        rows_input.append(dict(rh_u=42, rh_u_wrt_ice_or_water=43))
+        rows_expected.append(dict(rh_u=42, rh_u_wrt_ice_or_water=43))
+        # rh_u is below range, but rh_u_wrt_ice_or_water is within range. Both should be flagged due to the OOL relationship
+        rows_input.append(dict(rh_u=-10, rh_u_wrt_ice_or_water=3))
+        rows_expected.append(dict(rh_u=np.nan, rh_u_wrt_ice_or_water=np.nan))
+        # rh_u is within range, but rh_u_wrt_ice_or_water is below range; rh_u_wrt_ice_or_water should be flagged
+        rows_input.append(dict(rh_u=54, rh_u_wrt_ice_or_water=-4))
+        rows_expected.append(dict(rh_u=54, rh_u_wrt_ice_or_water=np.nan))
+        # rh_u is above range, but rh_u_wrt_ice_or_water is within range. Both should be flagged due to the OOL relationship
+        rows_input.append(dict(rh_u=160, rh_u_wrt_ice_or_water=120))
+        rows_expected.append(dict(rh_u=np.nan, rh_u_wrt_ice_or_water=np.nan))
+        # rh_u is within range, but rh_u_wrt_ice_or_water is above range; rh_u_wrt_ice_or_water should be flagged
+        rows_input.append(dict(rh_u=100, rh_u_wrt_ice_or_water=255))
+        rows_expected.append(dict(rh_u=100, rh_u_wrt_ice_or_water=np.nan))
 
         # Prepare the data
         df_input = pd.DataFrame(rows_input, dtype=float)