diff --git a/gnssanalysis/gn_io/sp3.py b/gnssanalysis/gn_io/sp3.py index 32c118d..6ab098e 100644 --- a/gnssanalysis/gn_io/sp3.py +++ b/gnssanalysis/gn_io/sp3.py @@ -53,7 +53,7 @@ _RE_SP3_HEAD_FDESCR = _re.compile(rb"\%c[ ]+(\w{1})[ ]+cc[ ](\w{3})") -_SP3_DEF_PV_WIDTH = [1, 3, 14, 14, 14, 14, 1, 2, 1, 2, 1, 2, 1, 3, 1, 1, 1, 1, 1, 1] +_SP3_DEF_PV_WIDTH = [1, 3, 14, 14, 14, 14, 1, 2, 1, 2, 1, 2, 1, 3, 1, 1, 1, 2, 1, 1] _SP3_DEF_PV_NAME = [ "PV_FLAG", "PRN", @@ -77,10 +77,79 @@ "Orbit_Pred_Flag", ] +SP3_POSITION_COLUMNS = [ + [ + "EST", + "EST", + "EST", + "EST", + "STD", + "STD", + "STD", + "STD", + "FLAGS", + "FLAGS", + "FLAGS", + "FLAGS", + ], + [ + "X", + "Y", + "Z", + "CLK", + "X", + "Y", + "Z", + "CLK", + "Clock_Event", + "Clock_Pred", + "Maneuver", + "Orbit_Pred", + ], +] + +SP3_VELOCITY_COLUMNS = [ + [ + "EST", + "EST", + "EST", + "EST", + "STD", + "STD", + "STD", + "STD", + "FLAGS", + "FLAGS", + "FLAGS", + "FLAGS", + ], + [ + "VX", + "VY", + "VZ", + "VCLOCK", + "VX", + "VY", + "VZ", + "VCLOCK", + "Clock_Event", + "Clock_Pred", + "Maneuver", + "Orbit_Pred", + ], +] + # Nodata ie NaN constants for SP3 format -SP3_CLOCK_NODATA_STRING = " 999999.999999" + +# NOTE: the CLOCK and POS NODATA strings below are technicaly incorrect. +# The specification requires a leading space on the CLOCK value, but Pandas DataFrame.to_string() (and others?) insist +# on adding a space between columns (so this cancels out the missing space here). +# For the POS value, no leading spaces are required by the SP3 spec, but we need the total width to be 13 chars, +# not 14 (the official width of the column i.e. F14.6), again because Pandas insists on adding a further space. +# See comment in gen_sp3_content() line ~622 for further discussion. +SP3_CLOCK_NODATA_STRING = "999999.999999" SP3_CLOCK_NODATA_NUMERIC = 999999 -SP3_POS_NODATA_STRING = " 0.000000" +SP3_POS_NODATA_STRING = " 0.000000" SP3_POS_NODATA_NUMERIC = 0 SP3_CLOCK_STD_NODATA = -1000 SP3_POS_STD_NODATA = -100 @@ -155,6 +224,12 @@ def _process_sp3_block( return _pd.DataFrame() epochs_dt = _pd.to_datetime(_pd.Series(date).str.slice(2, 21).values.astype(str), format=r"%Y %m %d %H %M %S") temp_sp3 = _pd.read_fwf(_io.StringIO(data), widths=widths, names=names) + # TODO set datatypes per column in advance + temp_sp3["Clock_Event_Flag"] = temp_sp3["Clock_Event_Flag"].fillna(" ") + temp_sp3["Clock_Pred_Flag"] = temp_sp3["Clock_Pred_Flag"].fillna(" ") + temp_sp3["Maneuver_Flag"] = temp_sp3["Maneuver_Flag"].fillna(" ") + temp_sp3["Orbit_Pred_Flag"] = temp_sp3["Orbit_Pred_Flag"].fillna(" ") + dt_index = _np.repeat(a=_gn_datetime.datetime2j2000(epochs_dt.values), repeats=len(temp_sp3)) temp_sp3.set_index(dt_index, inplace=True) temp_sp3.index.name = "J2000" @@ -216,26 +291,12 @@ def read_sp3( if pOnly or parsed_header.HEAD.loc["PV_FLAG"] == "P": sp3_df = sp3_df.loc[sp3_df.index.get_level_values("PV_FLAG") == "P"] sp3_df.index = sp3_df.index.droplevel("PV_FLAG") + # TODO consider exception handling if EP rows encountered else: position_df = sp3_df.xs("P", level="PV_FLAG") velocity_df = sp3_df.xs("V", level="PV_FLAG") - velocity_df.columns = [ - [ - "EST", - "EST", - "EST", - "EST", - "STD", - "STD", - "STD", - "STD", - "a1", - "a2", - "a3", - "a4", - ], - ["VX", "VY", "VZ", "VCLOCK", "VX", "VY", "VZ", "VCLOCK", "", "", "", ""], - ] + # TODO consider exception handling if EV rows encountered + velocity_df.columns = SP3_VELOCITY_COLUMNS sp3_df = _pd.concat([position_df, velocity_df], axis=1) # sp3_df.drop(columns="PV_FLAG", inplace=True) @@ -279,23 +340,7 @@ def _reformat_df(sp3_df: _pd.DataFrame) -> _pd.DataFrame: # remove PRN and PV_FLAG columns sp3_df = sp3_df.drop(columns=["PRN", "PV_FLAG"]) # rename columns x_coordinate -> [EST, X], y_coordinate -> [EST, Y] - sp3_df.columns = [ - [ - "EST", - "EST", - "EST", - "EST", - "STD", - "STD", - "STD", - "STD", - "a1", - "a2", - "a3", - "a4", - ], - ["X", "Y", "Z", "CLK", "X", "Y", "Z", "CLK", "", "", "", ""], - ] + sp3_df.columns = SP3_POSITION_COLUMNS return sp3_df @@ -551,6 +596,7 @@ def gen_sp3_content( # options that .sort_index() provides sp3_df = sp3_df.sort_index(ascending=True) out_df = sp3_df["EST"] + flags_df = sp3_df["FLAGS"] # Prediction, maneuver, etc. # If we have STD information transform it to the output format (integer exponents) and add to dataframe if "STD" in sp3_df: # In future we should pull this information from the header on read and store it in the dataframe attributes @@ -580,7 +626,7 @@ def clk_log(x): std_df.attrs = {} std_df = std_df.transform({"X": pos_log, "Y": pos_log, "Z": pos_log, "CLK": clk_log}) std_df = std_df.rename(columns=lambda x: "STD_" + x) - out_df = _pd.concat([out_df, std_df], axis="columns") + out_df = _pd.concat([out_df, std_df, flags_df], axis="columns") def prn_formatter(x): return f"P{x}" @@ -673,6 +719,8 @@ def clk_std_formatter(x): # relevant columns. # NOTE: NaN and infinity values do NOT invoke the formatter, though you can put a string in a primarily numeric # column, so we format the nodata values ahead of time, above. + # NOTE: you CAN'T mix datatypes as described above, in Pandas 3 and above, so this approach will need to be + # updated to use chained calls to format(). epoch_vals.to_string( buf=out_buf, index=False,