diff --git a/fink_fat/associations/association_orbit.py b/fink_fat/associations/association_orbit.py index 36c5c893..77f3ea8e 100644 --- a/fink_fat/associations/association_orbit.py +++ b/fink_fat/associations/association_orbit.py @@ -4,6 +4,7 @@ from fink_fat.command_line.orbit_cli import switch_local_cluster from fink_fat.others.utils import LoggerNewLine +from fink_fat.orbit_fitting.utils import orb_class def generate_fake_traj_id(orb_pdf: pd.DataFrame) -> Tuple[pd.DataFrame, dict]: @@ -40,7 +41,7 @@ def orbit_associations( last_night: str, logger: LoggerNewLine, verbose: bool, -) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: +) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.Series]: """ Add the alerts 5-flagged to the trajectory_df and recompute the orbit with the new points. @@ -61,8 +62,8 @@ def orbit_associations( Returns ------- - Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame] - the new orbits, the updated trajectories and the old orbits + Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.Series] + the new orbits, the updated trajectories and the old orbits, the id of the updated orbits """ if verbose: logger.info("start the associations with the orbits") @@ -78,7 +79,7 @@ def orbit_associations( .drop("roid", axis=1) ) if len(orbit_alert_assoc) == 0: - return orbits, trajectory_df, pd.DataFrame(columns=orbits.columns) + return orbits, trajectory_df, pd.DataFrame(columns=orbits.columns), pd.Series() updated_sso_id = orbit_alert_assoc["ssoCandId"].unique() # get the trajectories to update @@ -109,6 +110,7 @@ def orbit_associations( ) logger.newline(2) new_orbit_pdf["ssoCandId"] = new_orbit_pdf["trajectory_id"].map(trid_to_ssoid) + new_orbit_pdf = orb_class(new_orbit_pdf) updated_id = new_orbit_pdf["ssoCandId"] # update orbit @@ -124,4 +126,4 @@ def orbit_associations( old_traj = trajectory_df[~trajectory_df["ssoCandId"].isin(updated_id)] trajectory_df = pd.concat([old_traj, new_traj])[traj_cols_to_keep] - return orbits, trajectory_df, old_orbit + return orbits, trajectory_df, old_orbit, updated_id diff --git a/fink_fat/command_line/cli_main/fitroid.py b/fink_fat/command_line/cli_main/fitroid.py index 5e9cecc0..af1d86a8 100644 --- a/fink_fat/command_line/cli_main/fitroid.py +++ b/fink_fat/command_line/cli_main/fitroid.py @@ -26,7 +26,7 @@ import fink_fat.slack_bot.slack_bot as slack -def get_default_input(): +def get_default_input() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: """_summary_ Returns @@ -50,31 +50,33 @@ def get_default_input(): ] ) - orbits = pd.DataFrame( - columns=[ - "ref_epoch", - "a", - "e", - "i", - "long. node", - "arg. peric", - "mean anomaly", - "rms_a", - "rms_e", - "rms_i", - "rms_long. node", - "rms_arg. peric", - "rms_mean anomaly", - "chi_reduced", - "last_ra", - "last_dec", - "last_jd", - "last_mag", - "last_fid", - "ssoCandId", - ] - ) - return trajectory_orb, orbits + orb_cols = [ + "ref_epoch", + "a", + "e", + "i", + "long. node", + "arg. peric", + "mean anomaly", + "rms_a", + "rms_e", + "rms_i", + "rms_long. node", + "rms_arg. peric", + "rms_mean anomaly", + "chi_reduced", + "last_ra", + "last_dec", + "last_jd", + "last_mag", + "last_fid", + "ssoCandId", + "class", + ] + + orbits = pd.DataFrame(columns=orb_cols) + old_orbits = pd.DataFrame(columns=orb_cols) + return trajectory_orb, orbits, old_orbits def roid_flags(config: configparser.ConfigParser) -> Tuple[bool, list]: @@ -209,7 +211,8 @@ def fitroid_associations( ) ) - trajectory_orb, orbits = get_default_input() + trajectory_orb, orbits, old_orbits = get_default_input() + new_or_updated_orbits = [] # associations between trajectories with orbit and the new alerts associated with ephemeride if os.path.exists(path_orbit) and os.path.exists(path_trajectory_orb): @@ -217,9 +220,12 @@ def fitroid_associations( logger.info("orbits file detected, start the associations with the orbits") orbits = pd.read_parquet(path_orbit) trajectory_orb = pd.read_parquet(path_trajectory_orb) - orbits, trajectory_orb, old_orbits = orbit_associations( + + orbits, trajectory_orb, old_orbits, updated_ssocandid = orbit_associations( config, alerts_night, trajectory_orb, orbits, last_night, logger, True ) + new_or_updated_orbits += updated_ssocandid.tolist() + if os.path.exists(path_old_orbits): save_old_orbits = pd.read_parquet(path_old_orbits) old_orbits = pd.concat([save_old_orbits, old_orbits]) @@ -316,7 +322,7 @@ def fitroid_associations( if "updated" not in trajectory_df: trajectory_df["updated"] = "N" - trajectory_df, fit_roid_df, trajectory_orb, orbits = trcand_to_orbit( + trajectory_df, fit_roid_df, trajectory_orb, orbits, new_ssoCandId = trcand_to_orbit( config, trajectory_df, trajectory_orb, @@ -326,6 +332,7 @@ def fitroid_associations( logger, True, ) + new_or_updated_orbits += new_ssoCandId.tolist() nb_tr_before_tw = len(fit_roid_df) trajectory_df, fit_roid_df = time_window( @@ -368,7 +375,13 @@ def fitroid_associations( if post_on_slack: slack.post_assoc_on_slack( - last_night, statistic_string, trajectory_df, trajectory_orb + last_night, + statistic_string, + trajectory_df, + trajectory_orb, + orbits, + old_orbits, + new_or_updated_orbits, ) if arguments["--verbose"]: diff --git a/fink_fat/command_line/orbit_cli.py b/fink_fat/command_line/orbit_cli.py index 8c1d411d..529a99cc 100644 --- a/fink_fat/command_line/orbit_cli.py +++ b/fink_fat/command_line/orbit_cli.py @@ -12,6 +12,7 @@ ) from fink_fat.command_line.utils_cli import assig_tags import fink_fat.others.launch_spark as spark +from fink_fat.orbit_fitting.utils import orb_class def intro_reset_orbit(): # pragma: no cover @@ -224,7 +225,7 @@ def trcand_to_orbit( last_night: str, logger: LoggerNewLine, verbose: bool, -) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]: +) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.Series]: """ Compute and return the orbits for the trajectories with enough points found by the prediction functions @@ -249,11 +250,12 @@ def trcand_to_orbit( Returns ------- - Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame,] + Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.Series] * trajectories found by the prediction functions without those sent to the orbit fitting - * kalman filters without those with the associated trajectories sent to the orbit fitting + * polyfit functions without those with the associated trajectories sent to the orbit fitting * trajectories with the ones with orbits * the orbits parameters with the new ones + * the ssoCandId of the new orbits """ # get the trajectories large enough to go to the orbit fitting orbit_limit = int(config["SOLVE_ORBIT_PARAMS"]["orbfit_limit"]) @@ -261,7 +263,7 @@ def trcand_to_orbit( large_traj = traj_size[traj_size >= orbit_limit] if len(large_traj) == 0: - return trajectory_df, trparams_df, trajectory_orb, orbits + return trajectory_df, trparams_df, trajectory_orb, orbits, pd.Series() # to be send to the orbit fitting, # a trajectory must have a number of point above the orbit point limit set in the config file @@ -278,7 +280,7 @@ def trcand_to_orbit( if nb_traj_to_orb == 0: # no trajectory updated during this night to send to orbit fitting - return trajectory_df, trparams_df, trajectory_orb, orbits + return trajectory_df, trparams_df, trajectory_orb, orbits, pd.Series() year, month, day = last_night.split("-") @@ -300,6 +302,8 @@ def trcand_to_orbit( new_traj_id = new_orbits["trajectory_id"].values new_traj_orb = traj_to_orb[traj_to_orb["trajectory_id"].isin(new_traj_id)] new_orbits, new_traj_orb = assig_tags(new_orbits, new_traj_orb, len(orbits) + 1) + new_orbits = orb_class(new_orbits) + new_ssocandid = new_orbits["ssoCandId"] # add the new orbits and new trajectories with orbit orbits = pd.concat([orbits, new_orbits]) @@ -327,7 +331,7 @@ def trcand_to_orbit( # Remove the failed orbit trajectory_df = trajectory_df[~trajectory_df["trajectory_id"].isin(failed_orbit)] trparams_df = trparams_df[~trparams_df["trajectory_id"].isin(failed_orbit)] - return trajectory_df, trparams_df, trajectory_orb, orbits + return trajectory_df, trparams_df, trajectory_orb, orbits, new_ssocandid if __name__ == "__main__": diff --git a/fink_fat/slack_bot/slack_bot.py b/fink_fat/slack_bot/slack_bot.py index 3abec68f..63739c86 100644 --- a/fink_fat/slack_bot/slack_bot.py +++ b/fink_fat/slack_bot/slack_bot.py @@ -7,6 +7,7 @@ import io from fink_fat.others.utils import init_logging +from fink_fat.orbit_fitting.utils import best_orbits def init_slackbot() -> WebClient: @@ -21,15 +22,19 @@ def init_slackbot() -> WebClient: return client -def post_msg_on_slack(webclient: WebClient, msg: str): +def post_msg_on_slack(webclient: WebClient, msg: list): logging = init_logging() try: - webclient.chat_postMessage( - channel="#bot_asteroid_test", - text=msg, - blocks=[{"type": "section", "text": {"type": "mrkdwn", "text": msg}}], - ) - logging.info("Post msg on slack successfull") + for tmp_msg in msg: + webclient.chat_postMessage( + channel="#bot_asteroid_test", + text=tmp_msg, + blocks=[ + {"type": "section", "text": {"type": "mrkdwn", "text": tmp_msg}} + ], + ) + logging.info("Post msg on slack successfull") + time.sleep(1) except SlackApiError as e: if e.response["ok"] is False: logging.error("Post slack msg error", exc_info=1) @@ -40,8 +45,10 @@ def post_assoc_on_slack( statistic_string: str, trajectory_df: pd.DataFrame, trajectory_orb: pd.DataFrame, + orbits: pd.DataFrame, + old_orbits: pd.DataFrame, + ssoCandId: list, ): - def size_hist(data, title): plt.figure() plt.title(title) @@ -77,7 +84,44 @@ def size_hist(data, title): traj_size_perml = f"<{result['files'][0]['permalink']}|{' '}>" orb_size_perml = f"<{result['files'][1]['permalink']}|{' '}>" + best_orb_msg = [] + if len(ssoCandId) == 0: + best_orb_msg.append("No updated or new orbits during this night") + else: + best_orb = best_orbits(orbits) + tmp_msg = "### 10 best new or updated orbits of the night ###\n" + best_orb_msg.append(tmp_msg) + for i, (_, rows) in enumerate(best_orb.iterrows()): + tmp_best = "------------------------------------------------------\n" + + prev_orb = old_orbits[ + old_orbits["ssoCandId"] == rows["ssoCandId"] + ].sort_values(["ref_epoch"]) + if len(prev_orb) > 0: + tmp_best += f"{i+1}. Orbit updated: {rows['ssoCandId']}\n" + tmp_best += f"\tclass: {rows['class']} (previous class: {prev_orb.iloc[-1]['class']})\n" + else: + tmp_best += f"{i+1}. New orbit: {rows['ssoCandId']}\n" + tmp_best += f"\tclass: {rows['class']}\n" + + traj = trajectory_orb[ + trajectory_orb["ssoCandId"] == rows["ssoCandId"] + ].sort_values("jd") + obs_window = traj["jd"].values[-1] - traj["jd"].values[0] + last_magpsf = traj["magpsf"].values[-1] + last_sigmapsf = traj["sigmapsf"].values[-1] + + tmp_best += f"\tnumber of observations in the trajectory: {len(traj)}\n" + tmp_best += f"\ttrajectory time window: {obs_window:.4f} days\n" + tmp_best += f"\tlast magpsf: {last_magpsf} ± {last_sigmapsf}\n" + tmp_best += f"\t\ta = {rows['a']} ± {rows['rms_a']}\n" + tmp_best += f"\t\te = {rows['e']} ± {rows['rms_e']}\n" + tmp_best += f"\t\ti = {rows['i']} ± {rows['rms_i']}\n\n" + best_orb_msg.append(tmp_best) + + msg_list = [] slack_msg = f""" +============================================ FINK-FAT STATISTICS OF THE NIGHT: {last_night} {statistic_string} @@ -86,5 +130,5 @@ def size_hist(data, title): {orb_size_perml} """ - - post_msg_on_slack(slack_client, slack_msg) + msg_list += [slack_msg] + best_orb_msg + post_msg_on_slack(slack_client, msg_list)