diff --git a/.coveragerc b/.coveragerc index b408c25..f9a5779 100644 --- a/.coveragerc +++ b/.coveragerc @@ -12,3 +12,4 @@ omit= $PWD/fink_filters/early_sn_candidates/* $PWD/fink_filters/filter_rate_based_kn_candidates/* $PWD/fink_filters/filter_early_kn_candidates/filter_utils.py + $PWD/fink_filters/filter_known_tde/resolver.py diff --git a/datatest_tde/tde_ZTF20abfcszi_202008.parquet b/datatest_tde/tde_ZTF20abfcszi_202008.parquet new file mode 100644 index 0000000..b5590fd Binary files /dev/null and b/datatest_tde/tde_ZTF20abfcszi_202008.parquet differ diff --git a/fink_filters/__init__.py b/fink_filters/__init__.py index 89455bf..280efeb 100644 --- a/fink_filters/__init__.py +++ b/fink_filters/__init__.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.21" +__version__ = "3.22" diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py index fc38cbb..69fd490 100755 --- a/fink_filters/filter_anomaly_notification/filter.py +++ b/fink_filters/filter_anomaly_notification/filter.py @@ -115,8 +115,8 @@ def anomaly_notification_( ... send_to_tg=False, channel_id=None, ... send_to_slack=False, channel_name=None) >>> print(pdf_anomalies['objectId'].values) - ['ZTF21acoshvy' 'ZTF18aapgymv' 'ZTF19aboujyi' 'ZTF18abgjtxx' 'ZTF18aaypnnd' - 'ZTF18abbtxsx' 'ZTF18aaakhsv' 'ZTF18actxdmj' 'ZTF18aapoack' 'ZTF18abzvnya'] + ['ZTF21acoshvy' 'ZTF18aaypnnd' 'ZTF18abgjtxx' 'ZTF18aapgymv' 'ZTF18aaakhsv' + 'ZTF18abhxigz' 'ZTF18aapoack' 'ZTF19aboujyi' 'ZTF18abbtxsx' 'ZTF18actxdmj'] """ # Filtering by coordinates if cut_coords: diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py index 55d616f..af64b31 100755 --- a/fink_filters/filter_anomaly_notification/filter_utils.py +++ b/fink_filters/filter_anomaly_notification/filter_utils.py @@ -83,7 +83,10 @@ def get_data_permalink_slack(ztf_id): ''' cutout = get_cutout(ztf_id) curve = get_curve(ztf_id) - slack_client = WebClient(os.environ['ANOMALY_SLACK_TOKEN']) + if 'ANOMALY_SLACK_TOKEN' in os.environ: + slack_client = WebClient(os.environ['ANOMALY_SLACK_TOKEN']) + else: + raise KeyError("You need to set up ANOMALY_SLACK_TOKEN in your .bashrc") try: curve.seek(0) cutout.seek(0) diff --git a/fink_filters/filter_known_tde/__init__.py b/fink_filters/filter_known_tde/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fink_filters/filter_known_tde/data/TDElist_Gezari2021.tsv b/fink_filters/filter_known_tde/data/TDElist_Gezari2021.tsv new file mode 100644 index 0000000..c193b4a --- /dev/null +++ b/fink_filters/filter_known_tde/data/TDElist_Gezari2021.tsv @@ -0,0 +1,56 @@ +NGC 5905 ROSAT X 0.01124 40.94 5.84 Bade et al. 1996 +RX J1624+75 ROSAT X 0.0636 43.38 06.05 Grupe et al. 1999 +RX J1242-11A ROSAT X 0.050 42.60 5.84 Komossa & Greiner 1999 +RX J1420+53 ROSAT X 0.147 43.38 5.67 Greiner et al. 2000 +GALEX D3-13 GALEX U 0.3698 43.98 4.66 Gezari et al. 2006 +SDSS J1323+48 XMM X 0.0875 44.30 5.91 Esquej et al. 2007 +GALEX D1-9 GALEX U 0.326 43.48 4.59 Gezari et al. 2008 +TDXF J1347-32 ROSAT X 0.0366 42.73 6.14 Cappelluti et al. 2009 +GALEX D23H-1 GALEX U 0.1855 43.95 4.70 Gezari et al. 2009 +SDSS J1311-01 Chandra X 0.195 41.74 6.14 Maksym, Ulmer & Eracleous 2010 +SwiftJ1644 Swift G 0.353 ND ND Bloom et al. 2011 +2XMMi J1847-63 XMM X 0.0353 42.82 5.96 Lin et al. 2011 +SDSS-TDE1 SDSS O 0.136 43.64 4.42 van Velzen et al. 2011 +SDSS-TDE2 SDSS O 0.2515 44.54 4.37 van Velzen et al. 2011 +PS1-10jh PS O 0.1696 44.47 4.59 Gezari et al. 2012 +SDSS J1201+30 XMM X 0.146 45.00 06.06 Saxton et al. 2012 +SwiftJ2058 Swift G 1.186 ND ND Cenko et al. 2012b +WINGS J1348+26 Chandra X 0.0651 41.79 06.06 Maksym et al. 2013 +PS1-11af PS O 0.4046 44.16 4.28 Chornock et al. 2014 +RBS 1032 ROSAT X 0.026 41.70 6.11 Maksym et al. 2014b +PTF-09ge PTF O 0.064 44.04 04.08 Arcavi et al. 2014 +PTF-09axc PTF O 0.1146 43.46 04.08 Arcavi et al. 2014 +PTF-09djl PTF O 0.184 44.42 4.41 Arcavi et al. 2014 +ASASSN-14ae ASAS-SN O 0.0436 43.87 4.29 Holoien et al. 2014 +3XMM J1521+07 XMM X 0.179 43.51 6.30 Lin et al. 2015 +SwiftJ1112 Swift G 0.89 ND ND Brown et al. 2015 +ASASSN-14li ASAS-SN O 0.02058 43.66 4.52 Holoien et al. 2016b +ASASSN-15lh ASAS-SN O 0.2326 45.34 4.30 Dong et al. 2016 +ASASSN-15oi ASAS-SN O 0.0484 44.45 4.60 Holoien et al. 2016b +iPTF-16axa PTF O 0.108 43.82 4.46 Hung et al. 2017 +iPTF-16fnl PTF O 0.0163 43.18 4.47 Blagorodnova et al. 2017 +3XMM J1500+01 XMM X 0.1454 43.08 06.06 Lin et al. 2017 +OGLE16aaa OGLE O 0.1655 44.22 4.36 Wyrzykowski et al. 2017 +XMMSL1 J0740-85 XMM X 0.0173 42.61 6.00 Saxton et al. 2017 +iPTF-15af PTF O 0.07897 44.10 4.85 Blagorodnova et al. 2019 +AT2017eqx/PS17dhz PS O 0.1089 43.82 4.32 Nicholl et al. 2019 +AT2018zr/PS18kh PS O 0.071 43.78 4.14 van Velzen et al. 2021 +AT2018bsi/ZTF18aahqkbt ZTF O 0.051 43.87 4.53 van Velzen et al. 2021 +AT2018dyb/ASASSN-18pg ASAS-SN O 0.018 44.08 4.40 Leloudas et al. 2019 +AT2018fyk/ASASSN-18ul ASAS-SN O 0.059 44.48 4.54 Wevers et al. 2019 +AT2018hco/ATLAS18way ATLAS O 0.088 44.25 4.39 van Velzen et al. 2021 +AT2018hyz/ASASSN-18zj ASAS-SN O 0.04573 44.10 4.25 Gomez et al. 2020 +AT2018iih/ATLAS18yzs ATLAS O 0.212 44.62 4.23 van Velzen et al. 2021 +AT2018lna/ZTF19aabbnzo ZTF O 0.091 44.56 4.59 van Velzen et al. 2021 +AT2018lni/ZTF18actaqdw ZTF O 0.138 44.21 4.38 van Velzen et al. 2021 +AT2019ahk/ASASSN-19bt ASAS-SN O 0.0262 44.08 4.30 Holoien et al. 2019b +AT2019azh/ASASSN-19dj ASAS-SN O 0.0222 44.50 4.51 van Velzen et al. 2021 +AT2019bhf/ZTF19aakswrb ZTF O 0.1206 43.91 4.27 van Velzen et al. 2021 +AT2019cho/ZTF19aakiwze ZTF O 0.193 43.98 4.19 van Velzen et al. 2021 +AT2019dsg/ZTF19aapreis ZTF O 0.0512 44.26 4.59 van Velzen et al. 2021 +AT2019ehz/Gaia19bpt Gaia O 0.074 44.03 4.34 van Velzen et al. 2021 +AT2019eve/ZTF19aatylnl ZTF O 0.0813 43.14 04.06 van Velzen et al. 2021 +AT2019lwu/ZTF19abidbya ZTF O 0.117 43.60 4.14 van Velzen et al. 2021 +AT2019meg/ZTF19abhhjcc ZTF O 0.152 44.36 4.44 van Velzen et al. 2021 +AT2019mha/ATLAS19qqu ATLAS O 0.148 44.05 4.35 van Velzen et al. 2021 +AT2019qiz/ZTF19abzrhgq ZTF O 0.0151 43.44 4.27 van Velzen et al. 2021 \ No newline at end of file diff --git a/fink_filters/filter_known_tde/data/Table1_Hammerstein b/fink_filters/filter_known_tde/data/Table1_Hammerstein new file mode 100644 index 0000000..a9d50fe --- /dev/null +++ b/fink_filters/filter_known_tde/data/Table1_Hammerstein @@ -0,0 +1,65 @@ +Title: The Final Season Reimagined: 30 Tidal Disruption Events from the + ZTF-I Survey +Authors: Hammerstein E., van Velzen S., Gezari S., Cenko S.B., Yao Y., Ward C., + Frederick S., Villanueva N., Somalwar J.J., Graham M.J., Kulkarni S.R., + Stern D., Andreoni I., Bellm E.C., Dekany R., Dhawan S., Drake A.J., + Fremling C., Gatkine P., Groom S.L., Ho A.Y.Q., Kasliwal M.M., + Karambelkar V., Kool E.C., Masci F.J., Medford M.S., Perley D.A., + Purdum J., van Roestel J., Sharma Y., Sollerman J., Taggart K., Yan L. +Table: ZTF-I TDEs +================================================================================ +Byte-by-byte Description of file: apjaca283t1_mrt.txt +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 9 A9 --- IAU IAU name + 11- 22 A12 --- ZTF ZTF Name + 24- 35 A12 --- Disc Discovery Name + 37- 48 A12 --- GoT ZTF TDE Working Group nickname + 50- 79 A30 --- ONames Other name(s) + 81- 99 A19 --- Class First TDE classification (1) + 101-115 A15 --- Spec TDE spectral class, Section 2.3 + 117-124 A8 --- zspec Spectroscopic redshift (2) +-------------------------------------------------------------------------------- +Note (1): First TDE classification report, corresponding to either this + paper, to van Velzen et al. (2021) [2021ApJ...908....4V] or to the + following abbreviations. For AT2019azh see Hinkle et al. (2021) + [2021MNRAS.500.1673H] in addition to ATel #12568 -- + ATel = Astronomer’s Telegram, https://astronomerstelegram.org/); + AN = AstroNotes, https://www.wis-tns.org/astronotes; + TNSCR = TNS classification reports. +Note (2): Redshifts were determined using host galaxy stellar absorption + lines or narrow emission lines associated with star formation, namely Ca + II H and K or narrow H{alpha} emission. +-------------------------------------------------------------------------------- +AT2018zr ZTF18aabtxvd PS18kh Ned ATel #11444 TDE-H 0.075 +AT2018bsi ZTF18aahqkbt ZTF18aahqkbt Jon ATel #12035 TDE-H+He 0.051 +AT2018hco ZTF18abxftqm ATLAS18way Sansa ATel #12263 TDE-H 0.088 +AT2018iih ZTF18acaqdaa ATLAS18yzs Jorah Gaia18dpo 2021ApJ...908....4V TDE-He 0.212 +AT2018hyz ZTF18acpdvos ASASSN-18zj Gendry ATLAS18bafs ATel #12198 TDE-H+He 0.046 +AT2018lni ZTF18actaqdw ZTF18actaqdw Arya 2021ApJ...908....4V TDE-H+He 0.138 +AT2018lna ZTF19aabbnzo ZTF19aabbnzo Cersei ATel #12509 TDE-H+He 0.091 +AT2018jbv ZTF18acnbpmd ZTF18acnbpmd Samwell ATLAS19acl, PS19aoz This paper TDE-featureless 0.340 +AT2019cho ZTF19aakiwze ZTF19aakiwze Petyr 2021ApJ...908....4V TDE-H+He 0.193 +AT2019bhf ZTF19aakswrb ZTF19aakswrb Varys 2021ApJ...908....4V TDE-H+He 0.121 +AT2019azh ZTF17aaazdba ASASSN-19dj Jaime Gaia19bvo ATel #12568 * TDE-H+He 0.022 +AT2019dsg ZTF19aapreis ZTF19aapreis Bran ATLAS19kl ATel #12752 TDE-H+He 0.051 +AT2019ehz ZTF19aarioci Gaia19bpt Brienne ATel #12789 TDE-H 0.074 +AT2019mha ZTF19abhejal ATLAS19qqu Bronn 2021ApJ...908....4V TDE-H+He 0.148 +AT2019meg ZTF19abhhjcc ZTF19abhhjcc Margaery Gaia19dhd AN-2019-88 TDE-H 0.152 +AT2019lwu ZTF19abidbya ZTF19abidbya Robb ATLAS19rnz, PS19ega 2021ApJ...908....4V TDE-H 0.117 +AT2019qiz ZTF19abzrhgq ZTF19abzrhgq Melisandre ATLAS19vfr, Gaia19eks, PS19gdd ATel #13131 TDE-H+He 0.015 +AT2019teq ZTF19accmaxo ZTF19accmaxo Missandei TNSCR #7482 TDE-H+He 0.087 +AT2020pj ZTF20aabqihu ZTF20aabqihu Gilly ATLAS20cab TNSCR #7481 TDE-H+He 0.068 +AT2019vcb ZTF19acspeuw ZTF19acspeuw Tormund Gaia19feb, ATLAS19bcyz TNSCR #7078 TDE-H+He 0.088 +AT2020ddv ZTF20aamqmfk ATLAS20gee Shae ATel #13655 TDE-He 0.160 +AT2020ocn ZTF18aakelin ZTF18aakelin Podrick ATel #13859 TDE-He 0.070 +AT2020opy ZTF20abjwvae ZTF20abjwvae High Sparrow PS20fxm ATel #13944 TDE-H+He 0.159 +AT2020mot ZTF20abfcszi ZTF20abfcszi Pycelle Gaia20ead ATel #13944 TDE-H+He 0.070 +AT2020mbq ZTF20abefeab ZTF20abefeab Yara ATLAS20pfz, PS20grv This paper TDE-H 0.093 +AT2020qhs ZTF20abowque ZTF20abowque Loras ATLAS20upw, PS20krl This paper TDE-featureless 0.345 +AT2020riz ZTF20abrnwfc ZTF20abrnwfc Talisa PS20jop This paper TDE-featureless 0.435 +AT2020wey ZTF20acitpfz ZTF20acitpfz Roose ATLAS20belb, Gaia20fck TNSCR #7769 TDE-H+He 0.027 +AT2020zso ZTF20acqoiyt ZTF20acqoiyt Hodor ATLAS20bfok TNSCR #8025 TDE-H+He 0.057 +AT2020ysg ZTF20abnorit ZTF20abnorit Osha ATLAS20bjqp, PS21cru This paper TDE-featureless 0.277\nl + diff --git a/fink_filters/filter_known_tde/data/log.txt b/fink_filters/filter_known_tde/data/log.txt new file mode 100644 index 0000000..d3a9db2 --- /dev/null +++ b/fink_filters/filter_known_tde/data/log.txt @@ -0,0 +1,2 @@ +Not found: ['RX J1624+75', 'RX J1242-11A', 'RX J1420+53', 'GALEX D3-13', 'SDSS J1323+48', 'GALEX D1-9', 'TDXF J1347-32', 'GALEX D23H-1', 'SDSS J1311-01', 'SwiftJ1644', '2XMMi J1847-63', 'SDSS-TDE1', 'SDSS-TDE2', 'SDSS J1201+30', 'SwiftJ2058', 'WINGS J1348+26', 'PTF-09ge', 'PTF-09axc', 'PTF-09djl', '3XMM J1521+07', 'SwiftJ1112', 'iPTF-16axa', 'iPTF-16fnl', '3XMM J1500+01', 'XMMSL1 J0740-85', 'iPTF-15af'] +Number of TDE found: 44/70 diff --git a/fink_filters/filter_known_tde/data/resolver.py b/fink_filters/filter_known_tde/data/resolver.py new file mode 100644 index 0000000..4c6a942 --- /dev/null +++ b/fink_filters/filter_known_tde/data/resolver.py @@ -0,0 +1,61 @@ +import requests +import pandas as pd + +APIURL = 'https://fink-portal.org' + +data = {'name': [], 'ra': [], 'dec': []} + +# Hammerstein TDEs +TDEs_hammerstein = pd.read_fwf('Table1_Hammerstein', skiprows=34, header=None) +tns_names = TDEs_hammerstein[0].to_list() + +unknown = [] +for name in tns_names: + r = requests.post( + '{}/api/v1/resolver'.format(APIURL), + json={'resolver': 'simbad', 'name': name} + ) + if r.json() == []: + r = requests.post( + '{}/api/v1/resolver'.format(APIURL), + json={'resolver': 'tns', 'name': name} + ) + + if r.json() == []: + unknown.append(name) + else: + data['name'].append(r.json()[0]['oname']) + data['ra'].append(r.json()[0]['jradeg']) + data['dec'].append(r.json()[0]['jdedeg']) + # print(name, r.json()) + +# Gezari TDEs +pdf = pd.read_csv('TDElist_Gezari2021.tsv', sep='\t', header=None) +names = pdf[0].values + +for name in names: + if '/' in name: + name = name.split('/')[0] + r = requests.post( + '{}/api/v1/resolver'.format(APIURL), + json={'resolver': 'simbad', 'name': name} + ) + if r.json() == []: + r = requests.post( + '{}/api/v1/resolver'.format(APIURL), + json={'resolver': 'tns', 'name': name} + ) + + if r.json() == []: + unknown.append(name) + else: + data['name'].append(r.json()[0]['oname']) + data['ra'].append(r.json()[0]['jradeg']) + data['dec'].append(r.json()[0]['jdedeg']) + # print(name, r.json()) +print('Not found: {}'.format(unknown)) + +pdf_final = pd.DataFrame.from_dict(data) +pdf_final = pdf_final.drop_duplicates() +print('Number of TDE found: {}/{}'.format(len(pdf_final), len(pdf_final) + len(unknown))) +pdf_final.to_parquet('tde.parquet') diff --git a/fink_filters/filter_known_tde/data/tde.parquet b/fink_filters/filter_known_tde/data/tde.parquet new file mode 100644 index 0000000..cbf69eb Binary files /dev/null and b/fink_filters/filter_known_tde/data/tde.parquet differ diff --git a/fink_filters/filter_known_tde/filter.py b/fink_filters/filter_known_tde/filter.py new file mode 100644 index 0000000..68ca6c0 --- /dev/null +++ b/fink_filters/filter_known_tde/filter.py @@ -0,0 +1,144 @@ +# Copyright 2023 AstroLab Software +# Author: Julien Peloton +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pyspark.sql.functions import pandas_udf, PandasUDFType +from pyspark.sql.types import BooleanType + +from astropy.coordinates import SkyCoord +from astropy import units as u + +from fink_science.xmatch.utils import cross_match_astropy + +from fink_filters.filter_anomaly_notification.filter_utils import msg_handler_slack +from fink_filters.tester import spark_unit_tests + +import pandas as pd +import numpy as np +import os + +def known_tde_(objectId, ra, dec, radius_arcsec=pd.Series([5])) -> pd.Series: + """ Return alerts matching with known TDEs + + Parameters + ---------- + objectId: Pandas series + Colujmn containing ZTF object ID + ra: Pandas series + Column containing the RA values of alerts + dec: Pandas series + Column containing the DEC values of alerts + radius_arcsec: series + Radius for crossmatch, in arcsecond + + Returns + ---------- + out: pandas.Series of bool + Return a Pandas DataFrame with the appropriate flag: + false for bad alert, and true for good alert. + + Examples + ---------- + >>> pdf = pd.read_parquet('datatest_tde') + >>> classification = known_tde_( + ... pdf['objectId'], + ... pdf['candidate'].apply(lambda x: x['ra']), + ... pdf['candidate'].apply(lambda x: x['dec'])) + >>> print(np.sum(classification)) + 1 + + """ + curdir = os.path.dirname(os.path.abspath(__file__)) + tdes = pd.read_parquet(curdir + '/data/tde.parquet') + + catalog_tde = SkyCoord( + ra=np.array(tdes.ra, dtype=float) * u.degree, + dec=np.array(tdes.dec, dtype=float) * u.degree + ) + + pdf = pd.DataFrame( + { + 'ra': ra, + 'dec': dec, + 'candid': range(len(ra)), + 'objectId': objectId + } + ) + + catalog_ztf = SkyCoord( + ra=np.array(ra.values, dtype=float) * u.degree, + dec=np.array(dec.values, dtype=float) * u.degree + ) + + pdf_merge, mask, idx2 = cross_match_astropy( + pdf, catalog_ztf, catalog_tde, radius_arcsec=radius_arcsec + ) + + pdf_merge['match'] = False + pdf_merge.loc[mask, 'match'] = True + + pdf_merge['intname'] = 'Unknown' + pdf_merge.loc[mask, 'intname'] = [ + str(i).strip() for i in tdes['name'].astype(str).values[idx2] + ] + + if ('ANOMALY_SLACK_TOKEN' in os.environ) and ('GITHUB_ENV' not in os.environ): + # send to Slack recursively + for _, row in pdf_merge[mask].iterrows(): + slack_data = [] + t1 = f' associated with {row.intname}' + slack_data.append(f'''{t1}''') + + msg_handler_slack(slack_data, "known_tde_follow_up", init_msg='New TDE association!') + + return pdf_merge['match'] + + +@pandas_udf(BooleanType(), PandasUDFType.SCALAR) +def known_tde(objectId, ra, dec) -> pd.Series: + """ Pandas UDF for early_sn_candidates_ + + Parameters + ---------- + objectId: Pandas series + Column containing ZTF object ID + ra: Pandas series + Column containing the RA values of alerts + dec: Pandas series + Column containing the DEC values of alerts + + Returns + ---------- + out: pandas.Series of bool + Return a Pandas DataFrame with the appropriate flag: + false for bad alert, and true for good alert. + + Examples + ---------- + >>> from fink_utils.spark.utils import apply_user_defined_filter + >>> df = spark.read.format('parquet').load('datatest_tde') + >>> f = 'fink_filters.filter_known_tde.filter.known_tde' + >>> df = apply_user_defined_filter(df, f) + >>> print(df.count()) + 1 + """ + series = known_tde_(objectId, ra, dec) + return series + + +if __name__ == "__main__": + """ Execute the test suite """ + + # Run the test suite + globs = globals() + spark_unit_tests(globs)