check for WSI in csv2bufr and synop2bufr pipelines (#374)

* csv2bufr, plugin changed to skip non white listed stations. * apply WSI check to synop2bufr * fix ref * fix test data Co-authored-by: david-i-berry <[email protected]>
wmo-im · Jan 24, 2023 · 723437c · 723437c
1 parent e15d0d7
commit 723437c
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 1 deletion.
diff --git a/tests/data/observations/malawi/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv b/tests/data/observations/malawi/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv
@@ -2,4 +2,4 @@
 "TIMESTAMP","RECORD","WMO_Block","Station_ID","Station_Name","WMO_Station_Type","M_Year","M_Month","M_DayOfMonth","M_HourOfDay","M_Minutes","Latitude","Longitude","Elevation","BP_Elevation","BP","QNH","BP_Change","BP_Tendency","Temp_H","AirTempK","DewPointTempK","RH","Sun_hr","SunHrs","Sun_hr24","SunHrs24","Rain_H","Rain_hr","Rain_mm_Tot","Temp_hr24","Temp24T","AirTempMaxK","AirTempMinK","WSpeed_height","Wind_Type","Wind_Sig","Wind_T","WSpeed","WindDir","WSpeed10M_Avg","WindG_Sig","WindGust","Solar_hr","SlrJ","Solar_hr24","SlrJ24"
 "TS","RN","","","","","","","","","","","","","m","","hPa","Pa","","m","K","K","%","","hours","","hours","m","","mm","","","K","K","","","","","meters/second","degrees","m/s","","m/s","","J/m^2","","J/m^2"
 "","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","WVc","WVc","Smp","Smp","Smp","Smp","Tot","Smp","Smp"
-"2021-11-18 11:55:00",2328,6,"0-454-2-awstoleza","Toleza Farm","0",2021,11,18,9,55,-14.95,34.96,764,765,93868.82,102865.8,-229.2422,7,1.5,308.5,288.8,30.92,-1,0.29,-24,1.446,1.5,-1,0,-24,0,310.1,303.8,2,"0",2,-10,1.487,50.36,1.08,"None",2.55,-1,3.075586,-24,11.8543
+"2021-11-18 11:55:00",2328,6,"0-454-2-AWSTOLEZA","Toleza Farm","0",2021,11,18,9,55,-14.95,34.96,764,765,93868.82,102865.8,-229.2422,7,1.5,308.5,288.8,30.92,-1,0.29,-24,1.446,1.5,-1,0,-24,0,310.1,303.8,2,"0",2,-10,1.487,50.36,1.08,"None",2.55,-1,3.075586,-24,11.8543
diff --git a/wis2box-management/wis2box/data/csv2bufr.py b/wis2box-management/wis2box/data/csv2bufr.py
@@ -27,6 +27,7 @@
 from csv2bufr import MAPPINGS, transform as transform_csv
 
 from wis2box.data.base import BaseAbstractData
+from wis2box.metadata.station import get_valid_wsi
 
 LOGGER = logging.getLogger(__name__)
 
@@ -76,6 +77,10 @@ def transform(self, input_data: Union[Path, bytes],
         # convert to list
         LOGGER.debug('Iterating over BUFR messages')
         for item in results:
+            wsi = item['_meta']['properties']['wigos_station_identifier']
+            if get_valid_wsi(wsi) is None:
+                LOGGER.error(f'Station {wsi} not in station list, skipping')
+                continue
             LOGGER.debug('Setting obs date for filepath creation')
             identifier = item['_meta']['id']
             data_date = item['_meta']['properties']['datetime']

diff --git a/wis2box-management/wis2box/data/synop2bufr.py b/wis2box-management/wis2box/data/synop2bufr.py
@@ -27,6 +27,7 @@
 
 from wis2box.data.base import BaseAbstractData
 from wis2box.env import DATADIR
+from wis2box.metadata.station import get_valid_wsi
 
 LOGGER = logging.getLogger(__name__)
 
@@ -80,6 +81,10 @@ def transform(self, input_data: Union[Path, bytes],
         # convert to list
         LOGGER.debug('Iterating over BUFR messages')
         for item in results:
+            wsi = item['_meta']['properties']['wigos_station_identifier']
+            if get_valid_wsi(wsi) is None:
+                LOGGER.error(f'Station {wsi} not in station list, skipping')
+                continue
             LOGGER.debug('Setting obs date for filepath creation')
             identifier = item['_meta']['id']
             data_date = item['_meta']['properties']['datetime']