diff --git a/src/pypromice/process/L2toL3.py b/src/pypromice/process/L2toL3.py index ae553f3d..cd9e70fa 100755 --- a/src/pypromice/process/L2toL3.py +++ b/src/pypromice/process/L2toL3.py @@ -106,17 +106,14 @@ def gpsCoordinatePostprocessing(ds, var, config_folder='../aws-l0/metadata/stati # saving the static value of 'lat','lon' or 'alt' stored in attribute # as it might be the only coordinate available for certain stations (e.g. bedrock) var_out = var.replace('gps_','') - - if var_out == 'alt': - if 'altitude' in list(ds.attrs.keys()): - static_value = float(ds.attrs['altitude']) - else: - # print('no standard altitude for', ds.attrs['station_id']) - static_value = np.nan - elif var_out == 'lat': - static_value = float(ds.attrs['latitude']) - elif var_out == 'lon': - static_value = float(ds.attrs['longitude']) + coord_names = {'alt':'altitude', 'lat':'latitude','lon':'longitude'} + + if var_out+'_avg' in list(ds.attrs.keys()): + static_value = float(ds.attrs[var_out+'_avg']) + elif coord_names[var_out] in list(ds.attrs.keys()): + static_value = float(ds.attrs[coord_names[var_out]]) + else: + static_value = np.nan # if there is no gps observations, then we use the static value repeated # for each time stamp diff --git a/src/pypromice/process/write.py b/src/pypromice/process/write.py index 5f9e4593..e8d9e6a7 100644 --- a/src/pypromice/process/write.py +++ b/src/pypromice/process/write.py @@ -48,7 +48,7 @@ def prepare_and_write(dataset, outpath, vars_df=None, meta_dict=None, time='60mi if 'gps_lon' in d2.keys(): d2 = reformat_lon(d2) else: - logger.info('%s does not have gpd_lon'%name) + logger.info('%s does not have gps_lon'%name) # Add variable attributes and metadata if vars_df is None: @@ -290,19 +290,61 @@ def addMeta(ds, meta): ds.attrs['date_metadata_modified'] = ds.attrs['date_created'] ds.attrs['processing_level'] = ds.attrs['level'].replace('L','level ') + + if 'lat' in ds.keys(): + lat_min = ds['lat'].min().values + lat_max = ds['lat'].max().values + elif 'gps_lat' in ds.keys(): + lat_min = ds['gps_lat'].min().values + lat_max = ds['gps_lat'].max().values + elif 'latitude' in ds.attrs.keys(): + lat_min = ds.attrs['latitude'] + lat_max = ds.attrs['latitude'] + else: + lat_min =np.nan + lat_max = np.nan + + + if 'lon' in ds.keys(): + lon_min = ds['lon'].min().values + lon_max = ds['lon'].max().values + elif 'gps_lon' in ds.keys(): + lon_min = ds['gps_lon'].min().values + lon_max = ds['gps_lon'].max().values + elif 'longitude' in ds.attrs.keys(): + lon_min = ds.attrs['longitude'] + lon_max = ds.attrs['longitude'] + else: + lon_min =np.nan + lon_max = np.nan + + if 'alt' in ds.keys(): + alt_min = ds['alt'].min().values + alt_max = ds['alt'].max().values + elif 'gps_alt' in ds.keys(): + alt_min = ds['gps_alt'].min().values + alt_max = ds['gps_alt'].max().values + elif 'altitude' in ds.attrs.keys(): + alt_min = ds.attrs['altitude'] + alt_max = ds.attrs['altitude'] + else: + alt_min =np.nan + alt_max = np.nan + ds.attrs['geospatial_bounds'] = "POLYGON((" + \ - f"{ds['lat'].min().values} {ds['lon'].min().values}, " + \ - f"{ds['lat'].min().values} {ds['lon'].max().values}, " + \ - f"{ds['lat'].max().values} {ds['lon'].max().values}, " + \ - f"{ds['lat'].max().values} {ds['lon'].min().values}, " + \ - f"{ds['lat'].min().values} {ds['lon'].min().values}))" - - ds.attrs['geospatial_lat_min'] = str(ds['lat'].min().values) - ds.attrs['geospatial_lat_max'] = str(ds['lat'].max().values) - ds.attrs['geospatial_lon_min'] = str(ds['lon'].min().values) - ds.attrs['geospatial_lon_max'] = str(ds['lon'].max().values) - ds.attrs['geospatial_vertical_min'] = str(ds['alt'].min().values) - ds.attrs['geospatial_vertical_max'] = str(ds['alt'].max().values) + f"{lat_min} {lon_min}, " + \ + f"{lat_min} {lon_max}, " + \ + f"{lat_max} {lon_max}, " + \ + f"{lat_max} {lon_min}, " + \ + f"{lat_min} {lon_min}))" + + ds.attrs['geospatial_lat_min'] = str(lat_min) + ds.attrs['geospatial_lat_max'] = str(lat_max) + ds.attrs['geospatial_lon_min'] = str(lon_min) + ds.attrs['geospatial_lon_max'] = str(lon_max) + ds.attrs['geospatial_vertical_min'] = str(alt_min) + ds.attrs['geospatial_vertical_max'] = str(alt_max) + ds.attrs['geospatial_vertical_positive'] = 'up' ds.attrs['time_coverage_start'] = str(ds['time'][0].values) ds.attrs['time_coverage_end'] = str(ds['time'][-1].values) @@ -394,4 +436,4 @@ def reformat_lon(dataset, exempt=['UWN', 'Roof_GEUS', 'Roof_PROMICE']): if 'gps_lon' not in dataset.keys(): return dataset dataset['gps_lon'] = dataset['gps_lon'] * -1 - return dataset \ No newline at end of file + return dataset diff --git a/src/pypromice/qc/github_data_issues.py b/src/pypromice/qc/github_data_issues.py index fd4d96e0..41603ed8 100644 --- a/src/pypromice/qc/github_data_issues.py +++ b/src/pypromice/qc/github_data_issues.py @@ -65,10 +65,10 @@ def flagNAN(ds_in, for v in varlist: if v in list(ds.keys()): - logger.info(f'---> flagging {t0} {t1} {v}') + logger.debug(f'---> flagging {t0} {t1} {v}') ds[v] = ds[v].where((ds['time'] < t0) | (ds['time'] > t1)) else: - logger.info(f'---> could not flag {v} not in dataset') + logger.debug(f'---> could not flag {v} not in dataset') return ds @@ -206,13 +206,14 @@ def adjustData(ds, t1 = pd.to_datetime(t1, utc=True).tz_localize(None) index_slice = dict(time=slice(t0, t1)) - if len(ds_out[var].loc[index_slice].time.time) == 0: + logger.info(f'---> {t0} {t1} {var} {func} {val}') logger.info("Time range does not intersect with dataset") continue - logger.info(f'---> {t0} {t1} {var} {func} {val}') - + else: + logger.debug(f'---> {t0} {t1} {var} {func} {val}') + if func == "add": ds_out[var].loc[index_slice] = ds_out[var].loc[index_slice].values + val # flagging adjusted values diff --git a/src/pypromice/qc/persistence.py b/src/pypromice/qc/persistence.py index 5f5d55f4..963ff786 100644 --- a/src/pypromice/qc/persistence.py +++ b/src/pypromice/qc/persistence.py @@ -83,7 +83,7 @@ def persistence_qc( mask = mask & (df[v]<99) n_masked = mask.sum() n_samples = len(mask) - logger.info( + logger.debug( f"Applying persistent QC in {v}. Filtering {n_masked}/{n_samples} samples" ) # setting outliers to NaN @@ -96,7 +96,7 @@ def persistence_qc( n_masked = mask.sum() n_samples = len(mask) - logger.info( + logger.debug( f"Applying persistent QC in {v}. Filtering {n_masked}/{n_samples} samples" ) # setting outliers to NaN