Skip to content

Commit

Permalink
improve slack bot
Browse files Browse the repository at this point in the history
  • Loading branch information
@FusRoman
FusRoman committed Jan 18, 2024
1 parent c41f8a5 commit 2365bd7
Show file tree
Hide file tree
Showing 4 changed files with 114 additions and 51 deletions.
12 changes: 7 additions & 5 deletions fink_fat/associations/association_orbit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

from fink_fat.command_line.orbit_cli import switch_local_cluster
from fink_fat.others.utils import LoggerNewLine
from fink_fat.orbit_fitting.utils import orb_class


def generate_fake_traj_id(orb_pdf: pd.DataFrame) -> Tuple[pd.DataFrame, dict]:
Expand Down Expand Up @@ -40,7 +41,7 @@ def orbit_associations(
last_night: str,
logger: LoggerNewLine,
verbose: bool,
) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.Series]:
"""
Add the alerts 5-flagged to the trajectory_df and recompute the orbit with the new points.
Expand All @@ -61,8 +62,8 @@ def orbit_associations(
Returns
-------
Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]
the new orbits, the updated trajectories and the old orbits
Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.Series]
the new orbits, the updated trajectories and the old orbits, the id of the updated orbits
"""
if verbose:
logger.info("start the associations with the orbits")
Expand All @@ -78,7 +79,7 @@ def orbit_associations(
.drop("roid", axis=1)
)
if len(orbit_alert_assoc) == 0:
return orbits, trajectory_df, pd.DataFrame(columns=orbits.columns)
return orbits, trajectory_df, pd.DataFrame(columns=orbits.columns), pd.Series()

updated_sso_id = orbit_alert_assoc["ssoCandId"].unique()
# get the trajectories to update
Expand Down Expand Up @@ -109,6 +110,7 @@ def orbit_associations(
)
logger.newline(2)
new_orbit_pdf["ssoCandId"] = new_orbit_pdf["trajectory_id"].map(trid_to_ssoid)
new_orbit_pdf = orb_class(new_orbit_pdf)
updated_id = new_orbit_pdf["ssoCandId"]

# update orbit
Expand All @@ -124,4 +126,4 @@ def orbit_associations(
old_traj = trajectory_df[~trajectory_df["ssoCandId"].isin(updated_id)]

trajectory_df = pd.concat([old_traj, new_traj])[traj_cols_to_keep]
return orbits, trajectory_df, old_orbit
return orbits, trajectory_df, old_orbit, updated_id
73 changes: 43 additions & 30 deletions fink_fat/command_line/cli_main/fitroid.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import fink_fat.slack_bot.slack_bot as slack


def get_default_input():
def get_default_input() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
"""_summary_
Returns
Expand All @@ -50,31 +50,33 @@ def get_default_input():
]
)

orbits = pd.DataFrame(
columns=[
"ref_epoch",
"a",
"e",
"i",
"long. node",
"arg. peric",
"mean anomaly",
"rms_a",
"rms_e",
"rms_i",
"rms_long. node",
"rms_arg. peric",
"rms_mean anomaly",
"chi_reduced",
"last_ra",
"last_dec",
"last_jd",
"last_mag",
"last_fid",
"ssoCandId",
]
)
return trajectory_orb, orbits
orb_cols = [
"ref_epoch",
"a",
"e",
"i",
"long. node",
"arg. peric",
"mean anomaly",
"rms_a",
"rms_e",
"rms_i",
"rms_long. node",
"rms_arg. peric",
"rms_mean anomaly",
"chi_reduced",
"last_ra",
"last_dec",
"last_jd",
"last_mag",
"last_fid",
"ssoCandId",
"class",
]

orbits = pd.DataFrame(columns=orb_cols)
old_orbits = pd.DataFrame(columns=orb_cols)
return trajectory_orb, orbits, old_orbits


def roid_flags(config: configparser.ConfigParser) -> Tuple[bool, list]:
Expand Down Expand Up @@ -209,17 +211,21 @@ def fitroid_associations(
)
)

trajectory_orb, orbits = get_default_input()
trajectory_orb, orbits, old_orbits = get_default_input()
new_or_updated_orbits = []

# associations between trajectories with orbit and the new alerts associated with ephemeride
if os.path.exists(path_orbit) and os.path.exists(path_trajectory_orb):
if arguments["--verbose"]:
logger.info("orbits file detected, start the associations with the orbits")
orbits = pd.read_parquet(path_orbit)
trajectory_orb = pd.read_parquet(path_trajectory_orb)
orbits, trajectory_orb, old_orbits = orbit_associations(

orbits, trajectory_orb, old_orbits, updated_ssocandid = orbit_associations(
config, alerts_night, trajectory_orb, orbits, last_night, logger, True
)
new_or_updated_orbits += updated_ssocandid.tolist()

if os.path.exists(path_old_orbits):
save_old_orbits = pd.read_parquet(path_old_orbits)
old_orbits = pd.concat([save_old_orbits, old_orbits])
Expand Down Expand Up @@ -316,7 +322,7 @@ def fitroid_associations(

if "updated" not in trajectory_df:
trajectory_df["updated"] = "N"
trajectory_df, fit_roid_df, trajectory_orb, orbits = trcand_to_orbit(
trajectory_df, fit_roid_df, trajectory_orb, orbits, new_ssoCandId = trcand_to_orbit(
config,
trajectory_df,
trajectory_orb,
Expand All @@ -326,6 +332,7 @@ def fitroid_associations(
logger,
True,
)
new_or_updated_orbits += new_ssoCandId.tolist()

nb_tr_before_tw = len(fit_roid_df)
trajectory_df, fit_roid_df = time_window(
Expand Down Expand Up @@ -368,7 +375,13 @@ def fitroid_associations(

if post_on_slack:
slack.post_assoc_on_slack(
last_night, statistic_string, trajectory_df, trajectory_orb
last_night,
statistic_string,
trajectory_df,
trajectory_orb,
orbits,
old_orbits,
new_or_updated_orbits,
)

if arguments["--verbose"]:
Expand Down
16 changes: 10 additions & 6 deletions fink_fat/command_line/orbit_cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
)
from fink_fat.command_line.utils_cli import assig_tags
import fink_fat.others.launch_spark as spark
from fink_fat.orbit_fitting.utils import orb_class


def intro_reset_orbit(): # pragma: no cover
Expand Down Expand Up @@ -224,7 +225,7 @@ def trcand_to_orbit(
last_night: str,
logger: LoggerNewLine,
verbose: bool,
) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:
) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.Series]:
"""
Compute and return the orbits for the trajectories with enough points found by the prediction functions
Expand All @@ -249,19 +250,20 @@ def trcand_to_orbit(
Returns
-------
Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame,]
Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.Series]
* trajectories found by the prediction functions without those sent to the orbit fitting
* kalman filters without those with the associated trajectories sent to the orbit fitting
* polyfit functions without those with the associated trajectories sent to the orbit fitting
* trajectories with the ones with orbits
* the orbits parameters with the new ones
* the ssoCandId of the new orbits
"""
# get the trajectories large enough to go to the orbit fitting
orbit_limit = int(config["SOLVE_ORBIT_PARAMS"]["orbfit_limit"])
traj_size = trajectory_df["trajectory_id"].value_counts()
large_traj = traj_size[traj_size >= orbit_limit]

if len(large_traj) == 0:
return trajectory_df, trparams_df, trajectory_orb, orbits
return trajectory_df, trparams_df, trajectory_orb, orbits, pd.Series()

# to be send to the orbit fitting,
# a trajectory must have a number of point above the orbit point limit set in the config file
Expand All @@ -278,7 +280,7 @@ def trcand_to_orbit(

if nb_traj_to_orb == 0:
# no trajectory updated during this night to send to orbit fitting
return trajectory_df, trparams_df, trajectory_orb, orbits
return trajectory_df, trparams_df, trajectory_orb, orbits, pd.Series()

year, month, day = last_night.split("-")

Expand All @@ -300,6 +302,8 @@ def trcand_to_orbit(
new_traj_id = new_orbits["trajectory_id"].values
new_traj_orb = traj_to_orb[traj_to_orb["trajectory_id"].isin(new_traj_id)]
new_orbits, new_traj_orb = assig_tags(new_orbits, new_traj_orb, len(orbits) + 1)
new_orbits = orb_class(new_orbits)
new_ssocandid = new_orbits["ssoCandId"]

# add the new orbits and new trajectories with orbit
orbits = pd.concat([orbits, new_orbits])
Expand Down Expand Up @@ -327,7 +331,7 @@ def trcand_to_orbit(
# Remove the failed orbit
trajectory_df = trajectory_df[~trajectory_df["trajectory_id"].isin(failed_orbit)]
trparams_df = trparams_df[~trparams_df["trajectory_id"].isin(failed_orbit)]
return trajectory_df, trparams_df, trajectory_orb, orbits
return trajectory_df, trparams_df, trajectory_orb, orbits, new_ssocandid


if __name__ == "__main__":
Expand Down
64 changes: 54 additions & 10 deletions fink_fat/slack_bot/slack_bot.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import io

from fink_fat.others.utils import init_logging
from fink_fat.orbit_fitting.utils import best_orbits


def init_slackbot() -> WebClient:
Expand All @@ -21,15 +22,19 @@ def init_slackbot() -> WebClient:
return client


def post_msg_on_slack(webclient: WebClient, msg: str):
def post_msg_on_slack(webclient: WebClient, msg: list):
logging = init_logging()
try:
webclient.chat_postMessage(
channel="#bot_asteroid_test",
text=msg,
blocks=[{"type": "section", "text": {"type": "mrkdwn", "text": msg}}],
)
logging.info("Post msg on slack successfull")
for tmp_msg in msg:
webclient.chat_postMessage(
channel="#bot_asteroid_test",
text=tmp_msg,
blocks=[
{"type": "section", "text": {"type": "mrkdwn", "text": tmp_msg}}
],
)
logging.info("Post msg on slack successfull")
time.sleep(1)
except SlackApiError as e:
if e.response["ok"] is False:
logging.error("Post slack msg error", exc_info=1)
Expand All @@ -40,8 +45,10 @@ def post_assoc_on_slack(
statistic_string: str,
trajectory_df: pd.DataFrame,
trajectory_orb: pd.DataFrame,
orbits: pd.DataFrame,
old_orbits: pd.DataFrame,
ssoCandId: list,
):

def size_hist(data, title):
plt.figure()
plt.title(title)
Expand Down Expand Up @@ -77,7 +84,44 @@ def size_hist(data, title):
traj_size_perml = f"<{result['files'][0]['permalink']}|{' '}>"
orb_size_perml = f"<{result['files'][1]['permalink']}|{' '}>"

best_orb_msg = []
if len(ssoCandId) == 0:
best_orb_msg.append("No updated or new orbits during this night")
else:
best_orb = best_orbits(orbits)
tmp_msg = "### 10 best new or updated orbits of the night ###\n"
best_orb_msg.append(tmp_msg)
for i, (_, rows) in enumerate(best_orb.iterrows()):
tmp_best = "------------------------------------------------------\n"

prev_orb = old_orbits[
old_orbits["ssoCandId"] == rows["ssoCandId"]
].sort_values(["ref_epoch"])
if len(prev_orb) > 0:
tmp_best += f"{i+1}. Orbit updated: {rows['ssoCandId']}\n"
tmp_best += f"\tclass: {rows['class']} (previous class: {prev_orb.iloc[-1]['class']})\n"
else:
tmp_best += f"{i+1}. New orbit: {rows['ssoCandId']}\n"
tmp_best += f"\tclass: {rows['class']}\n"

traj = trajectory_orb[
trajectory_orb["ssoCandId"] == rows["ssoCandId"]
].sort_values("jd")
obs_window = traj["jd"].values[-1] - traj["jd"].values[0]
last_magpsf = traj["magpsf"].values[-1]
last_sigmapsf = traj["sigmapsf"].values[-1]

tmp_best += f"\tnumber of observations in the trajectory: {len(traj)}\n"
tmp_best += f"\ttrajectory time window: {obs_window:.4f} days\n"
tmp_best += f"\tlast magpsf: {last_magpsf} ± {last_sigmapsf}\n"
tmp_best += f"\t\ta = {rows['a']} ± {rows['rms_a']}\n"
tmp_best += f"\t\te = {rows['e']} ± {rows['rms_e']}\n"
tmp_best += f"\t\ti = {rows['i']} ± {rows['rms_i']}\n\n"
best_orb_msg.append(tmp_best)

msg_list = []
slack_msg = f"""
============================================
FINK-FAT STATISTICS OF THE NIGHT: {last_night}
{statistic_string}
Expand All @@ -86,5 +130,5 @@ def size_hist(data, title):
{orb_size_perml}
"""

post_msg_on_slack(slack_client, slack_msg)
msg_list += [slack_msg] + best_orb_msg
post_msg_on_slack(slack_client, msg_list)

0 comments on commit 2365bd7

Please sign in to comment.