diff --git a/tests/data/observations/malawi/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv b/tests/data/observations/malawi/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv index 6e3f92e8..8ccad38d 100644 --- a/tests/data/observations/malawi/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv +++ b/tests/data/observations/malawi/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv @@ -2,4 +2,4 @@ "TIMESTAMP","RECORD","WMO_Block","Station_ID","Station_Name","WMO_Station_Type","M_Year","M_Month","M_DayOfMonth","M_HourOfDay","M_Minutes","Latitude","Longitude","Elevation","BP_Elevation","BP","QNH","BP_Change","BP_Tendency","Temp_H","AirTempK","DewPointTempK","RH","Sun_hr","SunHrs","Sun_hr24","SunHrs24","Rain_H","Rain_hr","Rain_mm_Tot","Temp_hr24","Temp24T","AirTempMaxK","AirTempMinK","WSpeed_height","Wind_Type","Wind_Sig","Wind_T","WSpeed","WindDir","WSpeed10M_Avg","WindG_Sig","WindGust","Solar_hr","SlrJ","Solar_hr24","SlrJ24" "TS","RN","","","","","","","","","","","","","m","","hPa","Pa","","m","K","K","%","","hours","","hours","m","","mm","","","K","K","","","","","meters/second","degrees","m/s","","m/s","","J/m^2","","J/m^2" "","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","WVc","WVc","Smp","Smp","Smp","Smp","Tot","Smp","Smp" -"2021-11-18 11:55:00",2328,6,"0-454-2-awstoleza","Toleza Farm","0",2021,11,18,9,55,-14.95,34.96,764,765,93868.82,102865.8,-229.2422,7,1.5,308.5,288.8,30.92,-1,0.29,-24,1.446,1.5,-1,0,-24,0,310.1,303.8,2,"0",2,-10,1.487,50.36,1.08,"None",2.55,-1,3.075586,-24,11.8543 +"2021-11-18 11:55:00",2328,6,"0-454-2-AWSTOLEZA","Toleza Farm","0",2021,11,18,9,55,-14.95,34.96,764,765,93868.82,102865.8,-229.2422,7,1.5,308.5,288.8,30.92,-1,0.29,-24,1.446,1.5,-1,0,-24,0,310.1,303.8,2,"0",2,-10,1.487,50.36,1.08,"None",2.55,-1,3.075586,-24,11.8543 diff --git a/wis2box-management/wis2box/data/csv2bufr.py b/wis2box-management/wis2box/data/csv2bufr.py index 06bc197f..db63db46 100644 --- a/wis2box-management/wis2box/data/csv2bufr.py +++ b/wis2box-management/wis2box/data/csv2bufr.py @@ -27,6 +27,7 @@ from csv2bufr import MAPPINGS, transform as transform_csv from wis2box.data.base import BaseAbstractData +from wis2box.metadata.station import get_valid_wsi LOGGER = logging.getLogger(__name__) @@ -76,6 +77,10 @@ def transform(self, input_data: Union[Path, bytes], # convert to list LOGGER.debug('Iterating over BUFR messages') for item in results: + wsi = item['_meta']['properties']['wigos_station_identifier'] + if get_valid_wsi(wsi) is None: + LOGGER.error(f'Station {wsi} not in station list, skipping') + continue LOGGER.debug('Setting obs date for filepath creation') identifier = item['_meta']['id'] data_date = item['_meta']['properties']['datetime'] diff --git a/wis2box-management/wis2box/data/synop2bufr.py b/wis2box-management/wis2box/data/synop2bufr.py index b274785c..39e06121 100644 --- a/wis2box-management/wis2box/data/synop2bufr.py +++ b/wis2box-management/wis2box/data/synop2bufr.py @@ -27,6 +27,7 @@ from wis2box.data.base import BaseAbstractData from wis2box.env import DATADIR +from wis2box.metadata.station import get_valid_wsi LOGGER = logging.getLogger(__name__) @@ -80,6 +81,10 @@ def transform(self, input_data: Union[Path, bytes], # convert to list LOGGER.debug('Iterating over BUFR messages') for item in results: + wsi = item['_meta']['properties']['wigos_station_identifier'] + if get_valid_wsi(wsi) is None: + LOGGER.error(f'Station {wsi} not in station list, skipping') + continue LOGGER.debug('Setting obs date for filepath creation') identifier = item['_meta']['id'] data_date = item['_meta']['properties']['datetime']