From 580f860a7662c9df152f11f836f05f4ba367daa4 Mon Sep 17 00:00:00 2001 From: Julien Date: Wed, 8 Sep 2021 23:41:13 +0200 Subject: [PATCH] redesign the logic to compute Slack message only when needed --- .../filter_early_kn_candidates/filter.py | 274 +++++++++--------- fink_filters/filter_kn_candidates/filter.py | 265 ++++++++--------- 2 files changed, 273 insertions(+), 266 deletions(-) diff --git a/fink_filters/filter_early_kn_candidates/filter.py b/fink_filters/filter_early_kn_candidates/filter.py index 3a78609..5d3a9e2 100644 --- a/fink_filters/filter_early_kn_candidates/filter.py +++ b/fink_filters/filter_early_kn_candidates/filter.py @@ -137,7 +137,13 @@ def early_kn_candidates( f_kn = high_drb & high_classtar & new_detection f_kn = f_kn & cdsxmatch.isin(keep_cds) & not_ztf_sso_candidate - if f_kn.any(): + # check if we need slack redirection + need_slack1 = 'KNWEBHOOK' in os.environ + need_slack1 = 'KNWEBHOOK_FINK' in os.environ + need_slack3 = 'KNWEBHOOK_AMA_GALAXIES' in os.environ + need_slack4 = 'KNWEBHOOK_DWF' in os.environ + + if f_kn.any() and (need_slack1 or need_slack1 or need_slack3 or need_slack4): # load mangrove catalog curdir = os.path.dirname(os.path.abspath(__file__)) mangrove_path = curdir + '/../data/mangrove_filtered.csv' @@ -204,18 +210,20 @@ def early_kn_candidates( f_kn.loc[f_kn] = np.array(galaxy_matching, dtype=bool) - # check the nature of close objects in SDSS catalog - if f_kn.any(): + # check the nature of close objects in SDSS catalog no_star = [] for i in range(sum(f_kn)): pos = SkyCoord( ra=np.array(ra[f_kn])[i] * u.degree, dec=np.array(dec[f_kn])[i] * u.degree - ) + ) + # for a test on "many" objects, you may wait 1s to stay under the # query limit. - table = SDSS.query_region(pos, fields=['type'], - radius=5 * u.arcsec) + table = SDSS.query_region( + pos, fields=['type'], + radius=5 * u.arcsec + ) type_close_objects = [] if table is not None: type_close_objects = table['type'] @@ -227,7 +235,6 @@ def early_kn_candidates( ) f_kn.loc[f_kn] = np.array(no_star, dtype=bool) - if f_kn.any(): # Simplify notations b = gal.b.degree[f_kn] ra = Angle( @@ -254,98 +261,117 @@ def early_kn_candidates( err_mag = err_mag[f_kn] field = field[f_kn] - dict_filt = {1: 'g', 2: 'r'} - for i, alertID in enumerate(objectId[f_kn]): - # information to send - alert_text = """ - *New kilonova candidate:* - """.format(alertID, alertID) - time_text = """ - *Time:*\n- {} UTC\n - Time since first detection: {:.1f} hours - """.format(Time(jd[i], format='jd').iso, delta_jd_first[i] * 24) - measurements_text = """ - *Measurement (band {}):*\n- Apparent magnitude: {:.2f} ± {:.2f} - """.format(dict_filt[fid[i]], mag[i], err_mag[i]) - host_text = """ - *Presumed host galaxy:*\n- HyperLEDA Name: {:s}\n- 2MASS XSC Name: {:s}\n- Luminosity distance: ({:.2f} ± {:.2f}) Mpc\n- RA/Dec: {:.7f} {:+.7f}\n- log10(Stellar mass/Ms): {:.2f} + dict_filt = {1: 'g', 2: 'r'} + for i, alertID in enumerate(objectId[f_kn]): + # information to send + alert_text = """ + *New kilonova candidate:* + """.format(alertID, alertID) + time_text = """ + *Time:*\n- {} UTC\n - Time since first detection: {:.1f} hours + """.format(Time(jd[i], format='jd').iso, delta_jd_first[i] * 24) + measurements_text = """ + *Measurement (band {}):*\n- Apparent magnitude: {:.2f} ± {:.2f} + """.format(dict_filt[fid[i]], mag[i], err_mag[i]) + host_text = """ + *Presumed host galaxy:*\n- HyperLEDA Name: {:s}\n- 2MASS XSC Name: {:s}\n- Luminosity distance: ({:.2f} ± {:.2f}) Mpc\n- RA/Dec: {:.7f} {:+.7f}\n- log10(Stellar mass/Ms): {:.2f} + """.format( + pdf_mangrove.loc[host_galaxies[i], 'HyperLEDA_name'][2:-1], + pdf_mangrove.loc[host_galaxies[i], '2MASS_name'][2:-1], + pdf_mangrove.loc[host_galaxies[i], 'lum_dist'], + pdf_mangrove.loc[host_galaxies[i], 'dist_err'], + pdf_mangrove.loc[host_galaxies[i], 'ra'], + pdf_mangrove.loc[host_galaxies[i], 'dec'], + pdf_mangrove.loc[host_galaxies[i], 'stellarmass'], + ) + crossmatch_text = """ + *Cross-match: *\n- Alert-host distance: {:.2f} kpc\n- Absolute magnitude: {:.2f} """.format( - pdf_mangrove.loc[host_galaxies[i], 'HyperLEDA_name'][2:-1], - pdf_mangrove.loc[host_galaxies[i], '2MASS_name'][2:-1], - pdf_mangrove.loc[host_galaxies[i], 'lum_dist'], - pdf_mangrove.loc[host_galaxies[i], 'dist_err'], - pdf_mangrove.loc[host_galaxies[i], 'ra'], - pdf_mangrove.loc[host_galaxies[i], 'dec'], - pdf_mangrove.loc[host_galaxies[i], 'stellarmass'], - ) - crossmatch_text = """ - *Cross-match: *\n- Alert-host distance: {:.2f} kpc\n- Absolute magnitude: {:.2f} - """.format( - host_alert_separation[i] * pdf_mangrove.loc[ - host_galaxies[i], 'ang_dist'] * 1000, - abs_mag_candidate[i], - ) - radec_text = """ - *RA/Dec:*\n- [hours, deg]: {} {}\n- [deg, deg]: {:.7f} {:+.7f} - """.format(ra_formatted[i], dec_formatted[i], ra[i], dec[i]) - galactic_position_text = """ - *Galactic latitude:*\n- [deg]: {:.7f}""".format(b[i]) - tns_text = '*TNS:* '.format(ra[i], dec[i]) - # message formatting - blocks = [ - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": alert_text - }, - ] - }, - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": time_text - }, - { - "type": "mrkdwn", - "text": host_text - }, - { - "type": "mrkdwn", - "text": radec_text - }, - { - "type": "mrkdwn", - "text": crossmatch_text - }, - { - "type": "mrkdwn", - "text": galactic_position_text - }, - { - "type": "mrkdwn", - "text": measurements_text - }, - { - "type": "mrkdwn", - "text": tns_text - }, - ] - }, - ] - - # Standard channels - error_message = """ - {} is not defined as env variable - if an alert has passed the filter, - the message has not been sent to Slack - """ - for url_name in ['KNWEBHOOK', 'KNWEBHOOK_FINK']: - if (url_name in os.environ): + host_alert_separation[i] * pdf_mangrove.loc[ + host_galaxies[i], 'ang_dist'] * 1000, + abs_mag_candidate[i], + ) + radec_text = """ + *RA/Dec:*\n- [hours, deg]: {} {}\n- [deg, deg]: {:.7f} {:+.7f} + """.format(ra_formatted[i], dec_formatted[i], ra[i], dec[i]) + galactic_position_text = """ + *Galactic latitude:*\n- [deg]: {:.7f}""".format(b[i]) + tns_text = '*TNS:* '.format(ra[i], dec[i]) + # message formatting + blocks = [ + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": alert_text + }, + ] + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": time_text + }, + { + "type": "mrkdwn", + "text": host_text + }, + { + "type": "mrkdwn", + "text": radec_text + }, + { + "type": "mrkdwn", + "text": crossmatch_text + }, + { + "type": "mrkdwn", + "text": galactic_position_text + }, + { + "type": "mrkdwn", + "text": measurements_text + }, + { + "type": "mrkdwn", + "text": tns_text + }, + ] + }, + ] + + # Standard channels + error_message = """ + {} is not defined as env variable + if an alert has passed the filter, + the message has not been sent to Slack + """ + for url_name in ['KNWEBHOOK', 'KNWEBHOOK_FINK']: + if (url_name in os.environ): + requests.post( + os.environ[url_name], + json={ + 'blocks': blocks, + 'username': 'Cross-match-based kilonova bot' + }, + headers={'Content-Type': 'application/json'}, + ) + else: + log = logging.Logger('Kilonova filter') + log.warning(error_message.format(url_name)) + + # Send alerts to amateurs only on Friday + now = datetime.datetime.utcnow() + + # Monday is 1 and Sunday is 7 + is_friday = (now.isoweekday() == 5) + + if (np.abs(b[i]) > 20) & (mag[i] < 20) & is_friday & need_slack3: requests.post( - os.environ[url_name], + os.environ['KNWEBHOOK_AMA_GALAXIES'], json={ 'blocks': blocks, 'username': 'Cross-match-based kilonova bot' @@ -354,44 +380,22 @@ def early_kn_candidates( ) else: log = logging.Logger('Kilonova filter') - log.warning(error_message.format(url_name)) - - # Grandma amateur channel - ama_in_env = ('KNWEBHOOK_AMA_GALAXIES' in os.environ) - - # Send alerts to amateurs only on Friday - now = datetime.datetime.utcnow() + log.warning(error_message.format('KNWEBHOOK_AMA_GALAXIES')) - # Monday is 1 and Sunday is 7 - is_friday = (now.isoweekday() == 5) - - if (np.abs(b[i]) > 20) & (mag[i] < 20) & is_friday & ama_in_env: - requests.post( - os.environ['KNWEBHOOK_AMA_GALAXIES'], - json={ - 'blocks': blocks, - 'username': 'Cross-match-based kilonova bot' - }, - headers={'Content-Type': 'application/json'}, - ) - else: - log = logging.Logger('Kilonova filter') - log.warning(error_message.format('KNWEBHOOK_AMA_GALAXIES')) - - # DWF channel and requirements - dwf_ztf_fields = [1525, 530, 482, 1476, 388, 1433] - dwf_in_env = ('KNWEBHOOK_DWF' in os.environ) - if (int(field.values[i]) in dwf_ztf_fields) and dwf_in_env: - requests.post( - os.environ['KNWEBHOOK_DWF'], - json={ - 'blocks': blocks, - 'username': 'kilonova bot' - }, - headers={'Content-Type': 'application/json'}, - ) - else: - log = logging.Logger('Kilonova filter') - log.warning(error_message.format('KNWEBHOOK_DWF')) + # DWF channel and requirements + dwf_ztf_fields = [1525, 530, 482, 1476, 388, 1433] + dwf_in_env = ('KNWEBHOOK_DWF' in os.environ) + if (int(field.values[i]) in dwf_ztf_fields) and dwf_in_env: + requests.post( + os.environ['KNWEBHOOK_DWF'], + json={ + 'blocks': blocks, + 'username': 'kilonova bot' + }, + headers={'Content-Type': 'application/json'}, + ) + else: + log = logging.Logger('Kilonova filter') + log.warning(error_message.format('KNWEBHOOK_DWF')) return f_kn diff --git a/fink_filters/filter_kn_candidates/filter.py b/fink_filters/filter_kn_candidates/filter.py index 250f86a..facfbf4 100644 --- a/fink_filters/filter_kn_candidates/filter.py +++ b/fink_filters/filter_kn_candidates/filter.py @@ -108,7 +108,12 @@ def kn_candidates( f_kn = high_knscore & high_drb & high_classtar & new_detection f_kn = f_kn & small_detection_history & cdsxmatch.isin(keep_cds) - if f_kn.any(): + # check if we need slack redirection + need_slack1 = 'KNWEBHOOK' in os.environ + need_slack1 = 'KNWEBHOOK_FINK' in os.environ + need_slack3 = 'KNWEBHOOK_AMA_CL' in os.environ + + if f_kn.any() and (need_slack1 or need_slack1 or need_slack3): # Galactic latitude transformation b = SkyCoord( np.array(ra[f_kn], dtype=float), @@ -141,124 +146,143 @@ def kn_candidates( fid = np.array(fid.astype(int)[f_kn]) jd = np.array(jd)[f_kn] - dict_filt = {1: 'g', 2: 'r'} - for i, alertID in enumerate(objectId[f_kn]): - # Careful - Spark casts None as NaN! - maskNotNone = ~np.isnan(np.array(cmagpsfc[f_kn].values[i])) + dict_filt = {1: 'g', 2: 'r'} + for i, alertID in enumerate(objectId[f_kn]): + # Careful - Spark casts None as NaN! + maskNotNone = ~np.isnan(np.array(cmagpsfc[f_kn].values[i])) - # Time since last detection (independently of the band) - jd_hist_allbands = np.array(np.array(cjdc[f_kn])[i])[maskNotNone] - delta_jd_last = jd_hist_allbands[-1] - jd_hist_allbands[-2] + # Time since last detection (independently of the band) + jd_hist_allbands = np.array(np.array(cjdc[f_kn])[i])[maskNotNone] + delta_jd_last = jd_hist_allbands[-1] - jd_hist_allbands[-2] - filt = fid[i] - maskFilter = np.array(cfidc[f_kn].values[i]) == filt - m = maskNotNone * maskFilter - if sum(m) < 2: - continue - # DC mag (history + last measurement) - mag_hist, err_hist = np.array([ - dc_mag(k[0], k[1], k[2], k[3], k[4], k[5], k[6]) - for k in zip( - cfidc[f_kn].values[i][m][-2:], - cmagpsfc[f_kn].values[i][m][-2:], - csigmapsfc[f_kn].values[i][m][-2:], - cmagnrc[f_kn].values[i][m][-2:], - csigmagnrc[f_kn].values[i][m][-2:], - cmagzpscic[f_kn].values[i][m][-2:], - cisdiffposc[f_kn].values[i][m][-2:], - ) - ]).T + filt = fid[i] + maskFilter = np.array(cfidc[f_kn].values[i]) == filt + m = maskNotNone * maskFilter + if sum(m) < 2: + continue + # DC mag (history + last measurement) + mag_hist, err_hist = np.array([ + dc_mag(k[0], k[1], k[2], k[3], k[4], k[5], k[6]) + for k in zip( + cfidc[f_kn].values[i][m][-2:], + cmagpsfc[f_kn].values[i][m][-2:], + csigmapsfc[f_kn].values[i][m][-2:], + cmagnrc[f_kn].values[i][m][-2:], + csigmagnrc[f_kn].values[i][m][-2:], + cmagzpscic[f_kn].values[i][m][-2:], + cisdiffposc[f_kn].values[i][m][-2:], + ) + ]).T - # Grab the last measurement and its error estimate - mag = mag_hist[-1] - err_mag = err_hist[-1] + # Grab the last measurement and its error estimate + mag = mag_hist[-1] + err_mag = err_hist[-1] - # Compute rate only if more than 1 measurement available - if len(mag_hist) > 1: - jd_hist = cjdc[f_kn].values[i][m] + # Compute rate only if more than 1 measurement available + if len(mag_hist) > 1: + jd_hist = cjdc[f_kn].values[i][m] - # rate is between `last` and `last-1` measurements only - dmag = mag_hist[-1] - mag_hist[-2] - dt = jd_hist[-1] - jd_hist[-2] - rate = dmag / dt - error_rate = np.sqrt(err_hist[-1]**2 + err_hist[-2]**2) / dt + # rate is between `last` and `last-1` measurements only + dmag = mag_hist[-1] - mag_hist[-2] + dt = jd_hist[-1] - jd_hist[-2] + rate = dmag / dt + error_rate = np.sqrt(err_hist[-1]**2 + err_hist[-2]**2) / dt - # information to send - alert_text = """ - *New kilonova candidate:* - """.format(alertID, alertID) - knscore_text = "*Kilonova score:* {:.2f}".format(knscore[i]) - score_text = """ - *Other scores:*\n- Early SN Ia: {:.2f}\n- Ia SN vs non-Ia SN: {:.2f}\n- SN Ia and Core-Collapse vs non-SN: {:.2f} - """.format(rfscore[i], snn_snia_vs_nonia[i], snn_sn_vs_all[i]) - time_text = """ - *Time:*\n- {} UTC\n - Time since last detection: {:.1f} days\n - Time since first detection: {:.1f} days - """.format(Time(jd[i], format='jd').iso, delta_jd_last, delta_jd_first[i]) - measurements_text = """ - *Measurement (band {}):*\n- Apparent magnitude: {:.2f} ± {:.2f} \n- Rate: ({:.2f} ± {:.2f}) mag/day\n - """.format(dict_filt[fid[i]], mag, err_mag, rate, error_rate) - radec_text = """ - *RA/Dec:*\n- [hours, deg]: {} {}\n- [deg, deg]: {:.7f} {:+.7f} - """.format(ra_formatted[i], dec_formatted[i], ra[i], dec[i]) - galactic_position_text = """ - *Galactic latitude:*\n- [deg]: {:.7f}""".format(b[i]) + # information to send + alert_text = """ + *New kilonova candidate:* + """.format(alertID, alertID) + knscore_text = "*Kilonova score:* {:.2f}".format(knscore[i]) + score_text = """ + *Other scores:*\n- Early SN Ia: {:.2f}\n- Ia SN vs non-Ia SN: {:.2f}\n- SN Ia and Core-Collapse vs non-SN: {:.2f} + """.format(rfscore[i], snn_snia_vs_nonia[i], snn_sn_vs_all[i]) + time_text = """ + *Time:*\n- {} UTC\n - Time since last detection: {:.1f} days\n - Time since first detection: {:.1f} days + """.format(Time(jd[i], format='jd').iso, delta_jd_last, delta_jd_first[i]) + measurements_text = """ + *Measurement (band {}):*\n- Apparent magnitude: {:.2f} ± {:.2f} \n- Rate: ({:.2f} ± {:.2f}) mag/day\n + """.format(dict_filt[fid[i]], mag, err_mag, rate, error_rate) + radec_text = """ + *RA/Dec:*\n- [hours, deg]: {} {}\n- [deg, deg]: {:.7f} {:+.7f} + """.format(ra_formatted[i], dec_formatted[i], ra[i], dec[i]) + galactic_position_text = """ + *Galactic latitude:*\n- [deg]: {:.7f}""".format(b[i]) - tns_text = '*TNS:* '.format(ra[i], dec[i]) - # message formatting - blocks = [ - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": alert_text - }, - { - "type": "mrkdwn", - "text": knscore_text - } - ] - }, - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": time_text - }, - { - "type": "mrkdwn", - "text": score_text - }, - { - "type": "mrkdwn", - "text": radec_text - }, - { - "type": "mrkdwn", - "text": measurements_text - }, - { - "type": "mrkdwn", - "text": galactic_position_text - }, - { - "type": "mrkdwn", - "text": tns_text - }, - ] - }, - ] + tns_text = '*TNS:* '.format(ra[i], dec[i]) + # message formatting + blocks = [ + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": alert_text + }, + { + "type": "mrkdwn", + "text": knscore_text + } + ] + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": time_text + }, + { + "type": "mrkdwn", + "text": score_text + }, + { + "type": "mrkdwn", + "text": radec_text + }, + { + "type": "mrkdwn", + "text": measurements_text + }, + { + "type": "mrkdwn", + "text": galactic_position_text + }, + { + "type": "mrkdwn", + "text": tns_text + }, + ] + }, + ] + + error_message = """ + {} is not defined as env variable + if an alert has passed the filter, + the message has not been sent to Slack + """ + for url_name in ['KNWEBHOOK', 'KNWEBHOOK_FINK']: + if (url_name in os.environ): + requests.post( + os.environ[url_name], + json={ + 'blocks': blocks, + 'username': 'Classifier-based kilonova bot' + }, + headers={'Content-Type': 'application/json'}, + ) + else: + log = logging.Logger('Kilonova filter') + log.warning(error_message.format(url_name)) + + # Send alerts to amateurs only on Friday + now = datetime.datetime.utcnow() - error_message = """ - {} is not defined as env variable - if an alert has passed the filter, - the message has not been sent to Slack - """ - for url_name in ['KNWEBHOOK', 'KNWEBHOOK_FINK']: - if (url_name in os.environ): + # Monday is 1 and Sunday is 7 + is_friday = (now.isoweekday() == 5) + + if (np.abs(b[i]) > 20) & (mag < 20) & is_friday & need_slack3: requests.post( - os.environ[url_name], + os.environ['KNWEBHOOK_AMA_CL'], json={ 'blocks': blocks, 'username': 'Classifier-based kilonova bot' @@ -269,25 +293,4 @@ def kn_candidates( log = logging.Logger('Kilonova filter') log.warning(error_message.format(url_name)) - ama_in_env = ('KNWEBHOOK_AMA_CL' in os.environ) - - # Send alerts to amateurs only on Friday - now = datetime.datetime.utcnow() - - # Monday is 1 and Sunday is 7 - is_friday = (now.isoweekday() == 5) - - if (np.abs(b[i]) > 20) & (mag < 20) & is_friday & ama_in_env: - requests.post( - os.environ['KNWEBHOOK_AMA_CL'], - json={ - 'blocks': blocks, - 'username': 'Classifier-based kilonova bot' - }, - headers={'Content-Type': 'application/json'}, - ) - else: - log = logging.Logger('Kilonova filter') - log.warning(error_message.format(url_name)) - return f_kn