From c9342d6b1f7f7a1df725559f7f2db71a62cef858 Mon Sep 17 00:00:00 2001 From: Christopher Rabotin Date: Thu, 30 Nov 2023 13:53:29 -0700 Subject: [PATCH 1/5] Return residuals plots if requested --- python/nyx_space/plots/od.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/nyx_space/plots/od.py b/python/nyx_space/plots/od.py index e4936609..2aebfbaa 100644 --- a/python/nyx_space/plots/od.py +++ b/python/nyx_space/plots/od.py @@ -595,7 +595,7 @@ def plot_residuals( show=True, ): """ - Plot of residuals, with 3-σ lines + Plot of residuals, with 3-σ lines. Returns a tuple of the plots if show=False. """ try: @@ -621,6 +621,8 @@ def plot_residuals( plt_any = False + rtn_plots = [] + for col in df.columns: if col.startswith(kind): fig = go.Figure() @@ -689,10 +691,15 @@ def plot_residuals( if show: fig.show() + else: + rtn_plots += [fig] if not plt_any: raise ValueError(f"No columns ending with {kind} found -- nothing plotted") + if not show: + return rtn_plots + def plot_residual_histogram( df, title, kind="Prefit", copyright=None, html_out=None, show=True From ebf878d08a5027858cbc88110d6c13e78d9971c8 Mon Sep 17 00:00:00 2001 From: Christopher Rabotin Date: Thu, 30 Nov 2023 14:30:19 -0700 Subject: [PATCH 2/5] Create a basic measurement plot and rename the previous one to overlay_measurements --- python/nyx_space/plots/__init__.py | 4 +- python/nyx_space/plots/od.py | 85 +++++++++++++++++++++++++++--- python/nyx_space/plots/utils.py | 4 +- 3 files changed, 81 insertions(+), 12 deletions(-) diff --git a/python/nyx_space/plots/__init__.py b/python/nyx_space/plots/__init__.py index e379b932..8a817e5f 100644 --- a/python/nyx_space/plots/__init__.py +++ b/python/nyx_space/plots/__init__.py @@ -17,7 +17,7 @@ """ from .gauss_markov import plot_gauss_markov -from .od import plot_covar, plot_estimates, plot_measurements +from .od import plot_covar, plot_estimates, overlay_measurements from .traj import plot_traj, plot_ground_track, plot_traj_errors __all__ = [ @@ -27,5 +27,5 @@ "plot_traj", "plot_traj_errors", "plot_ground_track", - "plot_measurements", + "overlay_measurements", ] diff --git a/python/nyx_space/plots/od.py b/python/nyx_space/plots/od.py index 2aebfbaa..006aee99 100644 --- a/python/nyx_space/plots/od.py +++ b/python/nyx_space/plots/od.py @@ -17,6 +17,8 @@ """ import plotly.graph_objects as go +import plotly.express as px +from datetime import datetime from .utils import plot_with_error, plot_line, finalize_plot, colors @@ -28,6 +30,7 @@ import numpy as np from scipy.stats import norm +from nyx_space.time import Epoch def plot_estimates( dfs, @@ -247,11 +250,11 @@ def plot_estimates( if msr_df is not None: # Plot the measurements on both plots - pos_fig = plot_measurements( + pos_fig = overlay_measurements( msr_df, title, time_col_name, fig=pos_fig, show=False ) - vel_fig = plot_measurements( + vel_fig = overlay_measurements( msr_df, title, time_col_name, fig=vel_fig, show=False ) @@ -454,11 +457,11 @@ def plot_covar( if msr_df is not None: # Plot the measurements on both plots - pos_fig = plot_measurements( + pos_fig = overlay_measurements( msr_df, title, time_col_name, fig=pos_fig, show=False ) - vel_fig = plot_measurements( + vel_fig = overlay_measurements( msr_df, title, time_col_name, fig=vel_fig, show=False ) @@ -481,15 +484,19 @@ def plot_covar( return pos_fig, vel_fig -def plot_measurements( +def overlay_measurements( + fig, dfs, title, time_col_name="Epoch:Gregorian UTC", html_out=None, copyright=None, - fig=None, show=True, ): + """ + Given a plotly figure, overlay the measurements as shaded regions on top of the existing plot. + For a plot of measurements only, use `plot_measurements`. + """ if not isinstance(dfs, list): dfs = [dfs] @@ -571,7 +578,7 @@ def plot_measurements( line_width=0, ) - finalize_plot(fig, title, x_title, copyright, show) + finalize_plot(fig, title, x_title, None, copyright) if html_out: with open(html_out, "w") as f: @@ -673,7 +680,7 @@ def plot_residuals( if msr_df is not None: # Plot the measurements on both plots - fig = plot_measurements( + fig = overlay_measurements( msr_df, title, time_col_name, fig=fig, show=False ) @@ -744,3 +751,65 @@ def plot_residual_histogram( if show: fig.show() + +def plot_measurements( + df, + msr_type, + title=None, + time_col_name="Epoch:Gregorian UTC", + html_out=None, + copyright=None, + show=True, +): + """ + Plot the provided measurement type, fuzzy matching of the column name + """ + + msr_col_name = [col for col in df.columns if msr_type in col.lower()] + + if title is None: + # Build a title + station_names = ", ".join([name for name in df["Tracking device"].unique()]) + start = Epoch(df["Epoch:Gregorian UTC"].iloc[0]) + end = Epoch(df["Epoch:Gregorian UTC"].iloc[-1]) + arc_duration = end.timedelta(start) + title = f"Measurements from {station_names} spanning {start} to {end} ({arc_duration})" + + try: + orig_tim_col = df[time_col_name] + except KeyError: + # Find the time column + try: + col_name = [x for x in df.columns if x.startswith("Epoch")][0] + except IndexError: + raise KeyError("Could not find any Epoch column") + print(f"Could not find time column {time_col_name}, using `{col_name}`") + orig_tim_col = df[col_name] + + # Build a Python datetime column + pd_ok_epochs = [] + for epoch in orig_tim_col: + epoch = epoch.replace("UTC", "").strip() + if "." not in epoch: + epoch += ".0" + pd_ok_epochs += [datetime.fromisoformat(str(epoch).replace("UTC", "").strip())] + df["time_col"] = pd.Series(pd_ok_epochs) + x_title = "Epoch {}".format(time_col_name[-3:]) + + fig = px.scatter(df, x="time_col", y=msr_col_name, color="Tracking device") + + finalize_plot(fig, title, x_title, msr_col_name[0], copyright) + + if html_out: + with open(html_out, "w") as f: + f.write(fig.to_html()) + print(f"Saved HTML to {html_out}") + + if show: + fig.show() + else: + return fig + +if __name__ == "__main__": + df = pd.read_parquet("output_data/msr-2023-11-25T06-14-01.parquet") + plot_measurements(df, "range") \ No newline at end of file diff --git a/python/nyx_space/plots/utils.py b/python/nyx_space/plots/utils.py index 5de2f53c..d7fb721c 100644 --- a/python/nyx_space/plots/utils.py +++ b/python/nyx_space/plots/utils.py @@ -104,7 +104,7 @@ def _add_watermark(who): nyx_tpl.layout.annotations = [ dict( name="watermark", - text=f"Nyx Space 🄯 AGPLv3 {year}", + text=f"Powered by Nyx Space © {year}", opacity=0.75, font=dict(color="#3d84e8", size=12), xref="paper", @@ -201,7 +201,7 @@ def finalize_plot(fig, title, xtitle=None, ytitle=None, copyright=None): """ annotations = [dict(templateitemname="watermark")] - if copyright: + if copyright is not None: annotations += [ dict( templateitemname="watermark", From ce36b650a192559950692951a95e62e84a8849aa Mon Sep 17 00:00:00 2001 From: Christopher Rabotin Date: Thu, 30 Nov 2023 14:36:20 -0700 Subject: [PATCH 3/5] Manual conversion of Epoch to datetime Warning persists, Plotly does it when it sees that any column is a bunch of datetime objects --- python/nyx_space/plots/od.py | 16 ++++++++-------- python/nyx_space/plots/traj.py | 9 +++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/python/nyx_space/plots/od.py b/python/nyx_space/plots/od.py index 006aee99..c17537e1 100644 --- a/python/nyx_space/plots/od.py +++ b/python/nyx_space/plots/od.py @@ -96,8 +96,8 @@ def plot_estimates( epoch = epoch.replace("UTC", "").strip() if "." not in epoch: epoch += ".0" - pd_ok_epochs += [epoch] - time_col = pd.to_datetime(pd_ok_epochs) + pd_ok_epochs += [datetime.fromisoformat(str(epoch).replace("UTC", "").strip())] + time_col = pd.Series(pd_ok_epochs) x_title = "Epoch {}".format(time_col_name[-3:]) # Check that the requested covariance frame exists @@ -336,8 +336,8 @@ def plot_covar( epoch = epoch.replace("UTC", "").strip() if "." not in epoch: epoch += ".0" - pd_ok_epochs += [epoch] - time_col = pd.to_datetime(pd_ok_epochs) + pd_ok_epochs += [datetime.fromisoformat(str(epoch).replace("UTC", "").strip())] + time_col = pd.Series(pd_ok_epochs) x_title = "Epoch {}".format(time_col_name[-3:]) # Check that the requested covariance frame exists @@ -525,8 +525,8 @@ def overlay_measurements( epoch = epoch.replace("UTC", "").strip() if "." not in epoch: epoch += ".0" - pd_ok_epochs += [epoch] - time_col = pd.to_datetime(pd_ok_epochs) + pd_ok_epochs += [datetime.fromisoformat(str(epoch).replace("UTC", "").strip())] + time_col = pd.Series(pd_ok_epochs) x_title = "Epoch {}".format(time_col_name[-3:]) # Diff the epochs of the measurements to find when there is a start and end. @@ -622,8 +622,8 @@ def plot_residuals( epoch = epoch.replace("UTC", "").strip() if "." not in epoch: epoch += ".0" - pd_ok_epochs += [epoch] - time_col = pd.to_datetime(pd_ok_epochs) + pd_ok_epochs += [datetime.fromisoformat(str(epoch).replace("UTC", "").strip())] + time_col = pd.Series(pd_ok_epochs) x_title = "Epoch {}".format(time_col_name[-3:]) plt_any = False diff --git a/python/nyx_space/plots/traj.py b/python/nyx_space/plots/traj.py index 485d5df5..be5b672f 100644 --- a/python/nyx_space/plots/traj.py +++ b/python/nyx_space/plots/traj.py @@ -19,6 +19,7 @@ import plotly.graph_objects as go from plotly.subplots import make_subplots import pandas as pd +from datetime import datetime from .utils import ( radii, @@ -219,8 +220,8 @@ def plot_orbit_elements( epoch = epoch.replace("UTC", "").strip() if "." not in epoch: epoch += ".0" - pd_ok_epochs += [epoch] - df["Epoch"] = pd.to_datetime(pd_ok_epochs) + pd_ok_epochs += [datetime.fromisoformat(str(epoch).replace("UTC", "").strip())] + df["Epoch"] = pd.Series(pd_ok_epochs) if not isinstance(names, list): names = [names] @@ -317,8 +318,8 @@ def plot_traj_errors( epoch = epoch.replace("UTC", "").strip() if "." not in epoch: epoch += ".0" - pd_ok_epochs += [epoch] - df["Epoch"] = pd.to_datetime(pd_ok_epochs) + pd_ok_epochs += [datetime.fromisoformat(str(epoch).replace("UTC", "").strip())] + df["Epoch"] = pd.Series(pd_ok_epochs) if not isinstance(names, list): names = [names] From 6aeffff9d4435dda554cc6846364ec6a39dbe543 Mon Sep 17 00:00:00 2001 From: Christopher Rabotin Date: Thu, 30 Nov 2023 14:57:51 -0700 Subject: [PATCH 4/5] Fix overlay_measurement calls --- python/nyx_space/plots/__init__.py | 3 ++- python/nyx_space/plots/od.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/nyx_space/plots/__init__.py b/python/nyx_space/plots/__init__.py index 8a817e5f..7a83d165 100644 --- a/python/nyx_space/plots/__init__.py +++ b/python/nyx_space/plots/__init__.py @@ -17,7 +17,7 @@ """ from .gauss_markov import plot_gauss_markov -from .od import plot_covar, plot_estimates, overlay_measurements +from .od import plot_covar, plot_estimates, plot_measurements, overlay_measurements from .traj import plot_traj, plot_ground_track, plot_traj_errors __all__ = [ @@ -27,5 +27,6 @@ "plot_traj", "plot_traj_errors", "plot_ground_track", + "plot_measurements", "overlay_measurements", ] diff --git a/python/nyx_space/plots/od.py b/python/nyx_space/plots/od.py index c17537e1..15337a98 100644 --- a/python/nyx_space/plots/od.py +++ b/python/nyx_space/plots/od.py @@ -251,11 +251,11 @@ def plot_estimates( if msr_df is not None: # Plot the measurements on both plots pos_fig = overlay_measurements( - msr_df, title, time_col_name, fig=pos_fig, show=False + pos_fig, msr_df, title, time_col_name, show=False ) vel_fig = overlay_measurements( - msr_df, title, time_col_name, fig=vel_fig, show=False + vel_fig, msr_df, title, time_col_name, show=False ) if html_out: @@ -458,11 +458,11 @@ def plot_covar( if msr_df is not None: # Plot the measurements on both plots pos_fig = overlay_measurements( - msr_df, title, time_col_name, fig=pos_fig, show=False + pos_fig, msr_df, title, time_col_name, show=False ) vel_fig = overlay_measurements( - msr_df, title, time_col_name, fig=vel_fig, show=False + vel_fig, msr_df, title, time_col_name, show=False ) if html_out: @@ -681,7 +681,7 @@ def plot_residuals( if msr_df is not None: # Plot the measurements on both plots fig = overlay_measurements( - msr_df, title, time_col_name, fig=fig, show=False + fig, msr_df, title, time_col_name, show=False ) finalize_plot( From 8e01cf4841ea0ab5a6ff5874a7434d29a98dc1b2 Mon Sep 17 00:00:00 2001 From: Christopher Rabotin Date: Fri, 1 Dec 2023 08:33:50 -0700 Subject: [PATCH 5/5] Measurement plot now as a strip if no msr_type provided --- python/nyx_space/plots/od.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/python/nyx_space/plots/od.py b/python/nyx_space/plots/od.py index 15337a98..70c994d5 100644 --- a/python/nyx_space/plots/od.py +++ b/python/nyx_space/plots/od.py @@ -500,9 +500,6 @@ def overlay_measurements( if not isinstance(dfs, list): dfs = [dfs] - if fig is None: - fig = go.Figure() - color_values = list(colors.values()) station_colors = {} @@ -754,7 +751,7 @@ def plot_residual_histogram( def plot_measurements( df, - msr_type, + msr_type=None, title=None, time_col_name="Epoch:Gregorian UTC", html_out=None, @@ -762,10 +759,8 @@ def plot_measurements( show=True, ): """ - Plot the provided measurement type, fuzzy matching of the column name + Plot the provided measurement type, fuzzy matching of the column name, or plot all as a strip """ - - msr_col_name = [col for col in df.columns if msr_type in col.lower()] if title is None: # Build a title @@ -796,9 +791,14 @@ def plot_measurements( df["time_col"] = pd.Series(pd_ok_epochs) x_title = "Epoch {}".format(time_col_name[-3:]) - fig = px.scatter(df, x="time_col", y=msr_col_name, color="Tracking device") + if msr_type is None: + fig = px.strip(df, x="time_col", y="Tracking device", color="Tracking device") + finalize_plot(fig, title, x_title, "All tracking data", copyright) + else: + msr_col_name = [col for col in df.columns if msr_type in col.lower()] - finalize_plot(fig, title, x_title, msr_col_name[0], copyright) + fig = px.scatter(df, x="time_col", y=msr_col_name, color="Tracking device") + finalize_plot(fig, title, x_title, msr_col_name[0], copyright) if html_out: with open(html_out, "w") as f: @@ -809,7 +809,3 @@ def plot_measurements( fig.show() else: return fig - -if __name__ == "__main__": - df = pd.read_parquet("output_data/msr-2023-11-25T06-14-01.parquet") - plot_measurements(df, "range") \ No newline at end of file