NPI-3446 various bugfixes in sp3 parse and write:

- fixed incorrect width of unused columns between data flag values (2 instead of 1). This allows the second prediction flag to be read in.
- Consolidated definitions of columns for maintainability.
- Updated column/index definitions to clearly define a FLAGS index, and specify names of individual flag columns.
- Fixed SP3 output generator to access and output FLAGS data.
- Addressed misalignment bug caused by the combination of CLK nodata value width, and Pandas adding a space between columns. Also applied this fix to POS nodata value, as it appears the same issue would apply there.
- Added handling for missing flag values (conformance to empty strings to avoid outputting nans)
- Added placeholder TODOs for throwing exceptions if record types are encountered that we currently do not implement support for (EP, EV).

gnssanalysis/gn_io/sp3.py

@@ -53,7 +53,7 @@
 _RE_SP3_HEAD_FDESCR = _re.compile(rb"\%c[ ]+(\w{1})[ ]+cc[ ](\w{3})")
 
 
-_SP3_DEF_PV_WIDTH = [1, 3, 14, 14, 14, 14, 1, 2, 1, 2, 1, 2, 1, 3, 1, 1, 1, 1, 1, 1]
+_SP3_DEF_PV_WIDTH = [1, 3, 14, 14, 14, 14, 1, 2, 1, 2, 1, 2, 1, 3, 1, 1, 1, 2, 1, 1]
 _SP3_DEF_PV_NAME = [
     "PV_FLAG",
     "PRN",
@@ -77,10 +77,79 @@
     "Orbit_Pred_Flag",
 ]
 
+SP3_POSITION_COLUMNS = [
+    [
+        "EST",
+        "EST",
+        "EST",
+        "EST",
+        "STD",
+        "STD",
+        "STD",
+        "STD",
+        "FLAGS",
+        "FLAGS",
+        "FLAGS",
+        "FLAGS",
+    ],
+    [
+        "X",
+        "Y",
+        "Z",
+        "CLK",
+        "X",
+        "Y",
+        "Z",
+        "CLK",
+        "Clock_Event",
+        "Clock_Pred",
+        "Maneuver",
+        "Orbit_Pred",
+    ],
+]
+
+SP3_VELOCITY_COLUMNS = [
+    [
+        "EST",
+        "EST",
+        "EST",
+        "EST",
+        "STD",
+        "STD",
+        "STD",
+        "STD",
+        "FLAGS",
+        "FLAGS",
+        "FLAGS",
+        "FLAGS",
+    ],
+    [
+        "VX",
+        "VY",
+        "VZ",
+        "VCLOCK",
+        "VX",
+        "VY",
+        "VZ",
+        "VCLOCK",
+        "Clock_Event",
+        "Clock_Pred",
+        "Maneuver",
+        "Orbit_Pred",
+    ],
+]
+
 # Nodata ie NaN constants for SP3 format
-SP3_CLOCK_NODATA_STRING = " 999999.999999"
+
+# NOTE: the CLOCK and POS NODATA strings below are technicaly incorrect.
+# The specification requires a leading space on the CLOCK value, but Pandas DataFrame.to_string() (and others?) insist
+# on adding a space between columns (so this cancels out the missing space here).
+# For the POS value, no leading spaces are required by the SP3 spec, but we need the total width to be 13 chars,
+# not 14 (the official width of the column i.e. F14.6), again because Pandas insists on adding a further space.
+# See comment in gen_sp3_content() line ~622 for further discussion.
+SP3_CLOCK_NODATA_STRING = "999999.999999"
 SP3_CLOCK_NODATA_NUMERIC = 999999
-SP3_POS_NODATA_STRING = "      0.000000"
+SP3_POS_NODATA_STRING = "     0.000000"
 SP3_POS_NODATA_NUMERIC = 0
 SP3_CLOCK_STD_NODATA = -1000
 SP3_POS_STD_NODATA = -100
@@ -155,6 +224,12 @@ def _process_sp3_block(
         return _pd.DataFrame()
     epochs_dt = _pd.to_datetime(_pd.Series(date).str.slice(2, 21).values.astype(str), format=r"%Y %m %d %H %M %S")
     temp_sp3 = _pd.read_fwf(_io.StringIO(data), widths=widths, names=names)
+    # TODO set datatypes per column in advance
+    temp_sp3["Clock_Event_Flag"] = temp_sp3["Clock_Event_Flag"].fillna(" ")
+    temp_sp3["Clock_Pred_Flag"] = temp_sp3["Clock_Pred_Flag"].fillna(" ")
+    temp_sp3["Maneuver_Flag"] = temp_sp3["Maneuver_Flag"].fillna(" ")
+    temp_sp3["Orbit_Pred_Flag"] = temp_sp3["Orbit_Pred_Flag"].fillna(" ")
+
     dt_index = _np.repeat(a=_gn_datetime.datetime2j2000(epochs_dt.values), repeats=len(temp_sp3))
     temp_sp3.set_index(dt_index, inplace=True)
     temp_sp3.index.name = "J2000"
@@ -216,26 +291,12 @@ def read_sp3(
     if pOnly or parsed_header.HEAD.loc["PV_FLAG"] == "P":
         sp3_df = sp3_df.loc[sp3_df.index.get_level_values("PV_FLAG") == "P"]
         sp3_df.index = sp3_df.index.droplevel("PV_FLAG")
+        # TODO consider exception handling if EP rows encountered
     else:
         position_df = sp3_df.xs("P", level="PV_FLAG")
         velocity_df = sp3_df.xs("V", level="PV_FLAG")
-        velocity_df.columns = [
-            [
-                "EST",
-                "EST",
-                "EST",
-                "EST",
-                "STD",
-                "STD",
-                "STD",
-                "STD",
-                "a1",
-                "a2",
-                "a3",
-                "a4",
-            ],
-            ["VX", "VY", "VZ", "VCLOCK", "VX", "VY", "VZ", "VCLOCK", "", "", "", ""],
-        ]
+        # TODO consider exception handling if EV rows encountered
+        velocity_df.columns = SP3_VELOCITY_COLUMNS
         sp3_df = _pd.concat([position_df, velocity_df], axis=1)
 
     # sp3_df.drop(columns="PV_FLAG", inplace=True)
@@ -279,23 +340,7 @@ def _reformat_df(sp3_df: _pd.DataFrame) -> _pd.DataFrame:
     # remove PRN and PV_FLAG columns
     sp3_df = sp3_df.drop(columns=["PRN", "PV_FLAG"])
     # rename columns x_coordinate -> [EST, X], y_coordinate -> [EST, Y]
-    sp3_df.columns = [
-        [
-            "EST",
-            "EST",
-            "EST",
-            "EST",
-            "STD",
-            "STD",
-            "STD",
-            "STD",
-            "a1",
-            "a2",
-            "a3",
-            "a4",
-        ],
-        ["X", "Y", "Z", "CLK", "X", "Y", "Z", "CLK", "", "", "", ""],
-    ]
+    sp3_df.columns = SP3_POSITION_COLUMNS
     return sp3_df
 
 
@@ -551,6 +596,7 @@ def gen_sp3_content(
         # options that .sort_index() provides
         sp3_df = sp3_df.sort_index(ascending=True)
     out_df = sp3_df["EST"]
+    flags_df = sp3_df["FLAGS"]  # Prediction, maneuver, etc.
     # If we have STD information transform it to the output format (integer exponents) and add to dataframe
     if "STD" in sp3_df:
         # In future we should pull this information from the header on read and store it in the dataframe attributes
@@ -580,7 +626,7 @@ def clk_log(x):
         std_df.attrs = {}
         std_df = std_df.transform({"X": pos_log, "Y": pos_log, "Z": pos_log, "CLK": clk_log})
         std_df = std_df.rename(columns=lambda x: "STD_" + x)
-        out_df = _pd.concat([out_df, std_df], axis="columns")
+        out_df = _pd.concat([out_df, std_df, flags_df], axis="columns")
 
     def prn_formatter(x):
         return f"P{x}"
@@ -673,6 +719,8 @@ def clk_std_formatter(x):
         # relevant columns.
         # NOTE: NaN and infinity values do NOT invoke the formatter, though you can put a string in a primarily numeric
         # column, so we format the nodata values ahead of time, above.
+        # NOTE: you CAN'T mix datatypes as described above, in Pandas 3 and above, so this approach will need to be
+        # updated to use chained calls to format().
         epoch_vals.to_string(
             buf=out_buf,
             index=False,