trying to be specific with variables to drop on join_l3

src/pypromice/process/join_l3.py

@@ -105,8 +105,20 @@ def readNead(infile):
         # combining thermocouple and CS100 temperatures
         ds['TA1'] =  ds['TA1'].combine_first(ds['TA3'])
         ds['TA2'] =  ds['TA2'].combine_first(ds['TA4'])
-
+               
         ds=ds.rename(var_name)
+
+        standard_vars_to_drop = ["NR", "TA3", "TA4", "TA5", "HW1_adj_flag", 
+             "HW2_adj_flag", "OSWR_adj_flag", "NR_cor", "z_surf_1", "z_surf_1_adj_flag", 
+             "z_surf_2", "z_surf_2_adj_flag", "z_surf_combined", 
+            "TA2m", "RH2m", "VW10m", "SZA", "SAA", 
+            "depth_t_i_1", "depth_t_i_2", "depth_t_i_3", "depth_t_i_4", "depth_t_i_5", 
+            "depth_t_i_6", "depth_t_i_7", "depth_t_i_8", "depth_t_i_9", "depth_t_i_10", "t_i_10m"
+            ]
+
+        # Drop the variables if they are present in the dataset
+        ds = ds.drop_vars([var for var in standard_vars_to_drop if var in ds])
+
         ds=ds.rename({'timestamp':'time'})
     return ds
 
@@ -214,7 +226,14 @@ def join_l3(config_folder, site, folder_l3, folder_gcnet, outpath, variables, me
             logger.info(stid+' was listed as station but could not be found in '+folder_l3+' nor '+folder_gcnet)
             continue
 
-        l3, _ = loadArr(filepath, isNead)            
+        l3, _ = loadArr(filepath, isNead)    
+
+        # removing specific variable from a given file
+        specific_vars_to_drop = station_info.get("skipped_variables",[])
+        if len(specific_vars_to_drop)>0:
+            logger.info("Skipping %s from %s"%(specific_vars_to_drop, stid))
+            l3 = l3.drop_vars([var for var in specific_vars_to_drop if var in l3])
+
         list_station_data.append((l3, station_info))
 
     # Sort the list in reverse chronological order so that we start with the latest data
@@ -251,19 +270,7 @@ def join_l3(config_folder, site, folder_l3, folder_gcnet, outpath, variables, me
             for v in l3_merged.data_vars:
                 if  v not in l3.data_vars:
                     l3[v] = l3.t_u*np.nan
-
-            # if l3 (older data) has variables that does not have l3_merged (newer data)
-            # then they are removed from l3
-            list_dropped = []
-            for v in l3.data_vars:
-                if v not in l3_merged.data_vars:
-                    if v != 'z_stake':
-                       list_dropped.append(v)
-                       l3 = l3.drop(v)
-                    else:
-                       l3_merged[v] = ('time', l3_merged.t_u.data*np.nan)
-            logger.info('Unused variables in older dataset: '+' '.join(list_dropped))
-
+
             # saving attributes of station under an attribute called $stid
             st_attrs = l3_merged.attrs.get('stations_attributes', {})
             st_attrs[stid] = l3.attrs.copy()

src/pypromice/process/write.py

@@ -260,12 +260,16 @@ def addMeta(ds, meta):
     elif 'gps_lon' in ds.keys():
         # caluclating average coordinates based on the measured coords (can be gappy)
         for v in ['gps_lat','gps_lon','gps_alt']:
-            ds.attrs[v+'_avg'] = ds[v].mean().item()
+            if v in ds.keys():
+                ds.attrs[v+'_avg'] = ds[v].mean().item()
+            else:
+                ds.attrs[v+'_avg'] = np.nan
         # dropping the less accurate standard coordinates given in the 
         # raw or tx config files
         for v in ['latitude','longitude']:
             if v in ds.attrs.keys():
                 del ds.attrs[v]
+
     # Attribute convention for data discovery
     # https://wiki.esipfed.org/Attribute_Convention_for_Data_Discovery_1-3
 

src/pypromice/resources/variable_aliases_GC-Net.csv

@@ -47,9 +47,9 @@ t_i_11,
 tilt_x,
 tilt_y,
 rot,
-gps_lat,latitude
-gps_lon,longitude
-gps_alt,elevation
+lat,latitude
+lon,longitude
+alt,elevation
 gps_time,
 gps_geounit,
 gps_hdop,