diff --git a/src/pypromice/postprocess/bufr_utilities.py b/src/pypromice/postprocess/bufr_utilities.py index 00e036d8..8537e7f2 100644 --- a/src/pypromice/postprocess/bufr_utilities.py +++ b/src/pypromice/postprocess/bufr_utilities.py @@ -66,28 +66,81 @@ class BUFRVariables: """ - wmo_id: str + # Station type: "mobile" or "land" + # =============================== + # Fixed land station schema: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307080 + # Mobile station schema: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307090 + station_type: str + + # WMO station identifier + # Land stations: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/301090 + # Mobile stations: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/301092 + # ====================================================================================================== + wmo_id: str timestamp: datetime.datetime - relativeHumidity: float = attrs.field(converter=round_converter(0)) - airTemperature: float = attrs.field(converter=round_converter(1)) - pressure: float = attrs.field(converter=round_converter(1)) - windDirection: float = attrs.field(converter=round_converter(0)) - windSpeed: float = attrs.field(converter=round_converter(1)) - latitude: float = attrs.field(converter=round_converter(6)) - longitude: float = attrs.field(converter=round_converter(6)) + + # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/005001 + # Scale: 5, unit: degrees + # TODO: Test if eccodes does the rounding as well. The rounding is was 6 which is larger that the scale. + latitude: float = attrs.field(converter=round_converter(5)) + # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/006001 + # Scale: 5, unit: degrees + longitude: float = attrs.field(converter=round_converter(5)) + + # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007030 + # Scale: 1, unit: m heightOfStationGroundAboveMeanSeaLevel: float = attrs.field( - converter=round_converter(2) + converter=round_converter(1) ) - # + # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007031 + # Scale: 1, unit: m heightOfBarometerAboveMeanSeaLevel: float = attrs.field( - converter=round_converter(2), + converter=round_converter(1), ) + + # Pressure information + # ==================== + # Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302031 + # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007004 + # Scale: -1, unit: Pa + pressure: float = attrs.field(converter=round_converter(-1)) + # There are two other pressure variables in the template: 302001 and 010062. + + # Basic synoptic "instantaneous" data + # =================================== + # Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302035 + # This section only include the temperature and humidity data (302032). + # Precipitation and cloud data are currently ignored. + # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007032 + # Scale: 2, unit: m + # This is the first appearance of this variable id. heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformTempRH: float = attrs.field( - converter=round_converter(4), + converter=round_converter(2), ) + # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/012101 + # Scale: 2, unit: K + airTemperature: float = attrs.field(converter=round_converter(2)) + # There is also a Dewpoint temperature in this template: 012103 which is currently unused. + # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/012103 + # Scale: 0, unit: % + relativeHumidity: float = attrs.field(converter=round_converter(0)) + + # Basic synoptic "period" data + # ============================ + # Definition table: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302043 + # Wind data: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/302042 + # Wind direction: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/011001 + # Scale: 0, unit: degrees + windDirection: float = attrs.field(converter=round_converter(0)) + # Wind speed: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/011002 + # Scale: 1, unit: m/s + windSpeed: float = attrs.field(converter=round_converter(1)) + # https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_B/007032 + # Scale: 2, unit: m + # This is the 7th appearance of this variable id. heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformWSPD: float = attrs.field( - converter=round_converter(4) + converter=round_converter(2) ) def as_series(self) -> pd.Series: @@ -131,6 +184,7 @@ def __eq__(self, other: "BUFRVariables"): BUFR_TEMPLATES = { "mobile": { + # Template definition: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307090 "unexpandedDescriptors": (307090), # message template, "synopMobil" "edition": 4, # latest edition "masterTableNumber": 0, @@ -146,6 +200,7 @@ def __eq__(self, other: "BUFRVariables"): "compressedData": 0, }, "land": { + # Template definition: https://vocabulary-manager.eumetsat.int/vocabularies/BUFR/WMO/32/TABLE_D/307080 "unexpandedDescriptors": (307080), # message template, "synopLand" "edition": 4, # latest edition "masterTableNumber": 0, diff --git a/tests/unit/bufr_export/test_bufr_utilitites.py b/tests/unit/bufr_export/test_bufr_utilitites.py index 2b9a19b1..49ecc203 100644 --- a/tests/unit/bufr_export/test_bufr_utilitites.py +++ b/tests/unit/bufr_export/test_bufr_utilitites.py @@ -181,3 +181,37 @@ def test_nan_value_serialization(self): variables_src, variables_read, ) + + def test_precision(self): + """ + Test if the BUFRVariable rounding configurations aligns with the BUFR format. + + Use np.random.random() to generate high precision random values. + """ + variables_src = BUFRVariables( + wmo_id="04464", + station_type="mobile", + timestamp=datetime.datetime(2023, 12, 19, 10, 0), + relativeHumidity=np.random.random(), + airTemperature=np.random.random(), + pressure=1000*np.random.random(), + windDirection=np.random.random(), + windSpeed=np.random.random(), + latitude=np.random.random(), + longitude=np.random.random(), + heightOfStationGroundAboveMeanSeaLevel=np.random.random(), + heightOfBarometerAboveMeanSeaLevel=np.random.random(), + heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformTempRH=np.random.random(), + heightOfSensorAboveLocalGroundOrDeckOfMarinePlatformWSPD=np.random.random(), + ) + with tempfile.TemporaryFile("w+b") as fp: + write_bufr_message(variables=variables_src, file=fp) + fp.seek(0) + variables_read = read_bufr_message( + fp=fp, + ) + + self.assertEqual( + variables_src, + variables_read, + ) diff --git a/tests/unit/bufr_export/test_create_bufr_files.py b/tests/unit/bufr_export/test_create_bufr_files.py index f9cf935e..2cb25afc 100644 --- a/tests/unit/bufr_export/test_create_bufr_files.py +++ b/tests/unit/bufr_export/test_create_bufr_files.py @@ -32,7 +32,7 @@ def tearDown(self): def test_create_bufr_files(self): """ - Teste the creation of bufr files and their output folder structure. + Test the creation of bufr files and their output folder structure. It does not test the content of the bufr files. """ input_dir = self.temp_dir / "input" diff --git a/tests/unit/bufr_export/test_get_bufr.py b/tests/unit/bufr_export/test_get_bufr.py index 51b24837..2598be3f 100644 --- a/tests/unit/bufr_export/test_get_bufr.py +++ b/tests/unit/bufr_export/test_get_bufr.py @@ -54,7 +54,7 @@ def test_bufr_variables_gcnet(self): stid=station_configuration.stid, station_configuration=station_configuration, relativeHumidity=69.0, - airTemperature=256.0, + airTemperature=255.95, pressure=77300.0, windDirection=149.0, windSpeed=14.9, @@ -84,7 +84,7 @@ def test_bufr_variables_promice_v2(self): stid=station_configuration.stid, station_configuration=station_configuration, relativeHumidity=69.0, - airTemperature=256.0, + airTemperature=255.95, pressure=77300.0, windDirection=149.0, windSpeed=14.9, @@ -114,7 +114,7 @@ def test_bufr_variables_promice_v3(self): stid=station_configuration.stid, station_configuration=station_configuration, relativeHumidity=69.0, - airTemperature=256.0, + airTemperature=255.95, pressure=77300.0, windDirection=149.0, windSpeed=14.9, @@ -165,7 +165,7 @@ def test_bufr_variables_static_gps_elevation(self): station_type=station_config.station_type, timestamp=timestamp, relativeHumidity=1.0, - airTemperature=252.2, # Converted to kelvin + airTemperature=252.15, # Converted to kelvin pressure=199300.0, windDirection=32.0, windSpeed=5.3, diff --git a/tests/unit/bufr_export/test_get_bufr_integration.py b/tests/unit/bufr_export/test_get_bufr_integration.py index 40112ccb..8d8edbc2 100644 --- a/tests/unit/bufr_export/test_get_bufr_integration.py +++ b/tests/unit/bufr_export/test_get_bufr_integration.py @@ -141,7 +141,7 @@ def test_get_bufr_has_new_data(self): # Newest measurement in tx_l3_test1.csv: 2023-12-07 23:00:00 timestamp=datetime.datetime(2023, 12, 7, 23, 00), relativeHumidity=69, - airTemperature=256.0, + airTemperature=255.95, pressure=77300.0, windDirection=149, windSpeed=14.9, @@ -180,7 +180,7 @@ def test_get_bufr_has_new_data_dont_store_position(self): # Newest measurement in tx_l3_test1.csv: 2023-12-07 23:00:00 timestamp=datetime.datetime(2023, 12, 7, 23, 00), relativeHumidity=69, - airTemperature=256.0, + airTemperature=255.95, pressure=77300.0, windDirection=149, windSpeed=14.9, @@ -260,7 +260,7 @@ def test_get_bufr_includes_datasets_not_in_latests_timestamps(self): # Newest measurement in tx_l3_test1.csv: 2023-12-07 23:00:00 timestamp=datetime.datetime(2023, 12, 7, 23, 00), relativeHumidity=69, - airTemperature=256.0, + airTemperature=255.95, pressure=77300.0, windDirection=149, windSpeed=14.9, @@ -321,7 +321,7 @@ def test_invalid_value_at_last_index(self): # Newest measurement in tx_l3_test1.csv: 2023-12-07 23:00:00 timestamp=datetime.datetime(2023, 12, 7, 23, 00), relativeHumidity=69, - airTemperature=256.0, + airTemperature=255.95, pressure=np.nan, windDirection=149, windSpeed=14.9, @@ -459,7 +459,7 @@ def test_ignore_newer_data_than_now_input(self): # Newest measurement in tx_l3_test1.csv: 2023-12-07 23:00:00 but now_timestamp is 2023-12-06 timestamp=datetime.datetime(2023, 12, 6, 0, 0), relativeHumidity=82, - airTemperature=250.8, + airTemperature=250.85, pressure=77370.0, windDirection=153, windSpeed=10.4, @@ -500,7 +500,7 @@ def test_land_station_export(self): # Newest measurement in tx_l3_test1.csv: 2023-12-07 23:00:00 timestamp=datetime.datetime(2023, 12, 7, 23, 00), relativeHumidity=69, - airTemperature=256.0, + airTemperature=255.95, pressure=77300.0, windDirection=149, windSpeed=14.9,