Skip to content

Commit

Permalink
Filter for known TDE (#161)
Browse files Browse the repository at this point in the history 
* Initial filter for TDE

* PEP8

* Fix print order

* Fix typo

* Renamed files

* Add the Slack bot

* Bump to 3.22

* PEP8

* Add dataset for TDE

* Remove the Slack bot from the CI

* Remove the Slack bot from the CI
  • Loading branch information
@JulienPeloton
JulienPeloton authored Nov 6, 2023
1 parent fa39076 commit 22d1ac8
Show file tree
Hide file tree
Showing 12 changed files with 336 additions and 4 deletions.
1 change: 1 addition & 0 deletions .coveragerc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,4 @@ omit=
$PWD/fink_filters/early_sn_candidates/*
$PWD/fink_filters/filter_rate_based_kn_candidates/*
$PWD/fink_filters/filter_early_kn_candidates/filter_utils.py
$PWD/fink_filters/filter_known_tde/resolver.py
Binary file added datatest_tde/tde_ZTF20abfcszi_202008.parquet
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion fink_filters/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = "3.21"
__version__ = "3.22"
4 changes: 2 additions & 2 deletions fink_filters/filter_anomaly_notification/filter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,8 @@ def anomaly_notification_(
... send_to_tg=False, channel_id=None,
... send_to_slack=False, channel_name=None)
>>> print(pdf_anomalies['objectId'].values)
['ZTF21acoshvy' 'ZTF18aapgymv' 'ZTF19aboujyi' 'ZTF18abgjtxx' 'ZTF18aaypnnd'
'ZTF18abbtxsx' 'ZTF18aaakhsv' 'ZTF18actxdmj' 'ZTF18aapoack' 'ZTF18abzvnya']
['ZTF21acoshvy' 'ZTF18aaypnnd' 'ZTF18abgjtxx' 'ZTF18aapgymv' 'ZTF18aaakhsv'
'ZTF18abhxigz' 'ZTF18aapoack' 'ZTF19aboujyi' 'ZTF18abbtxsx' 'ZTF18actxdmj']
"""
# Filtering by coordinates
if cut_coords:
Expand Down
5 changes: 4 additions & 1 deletion fink_filters/filter_anomaly_notification/filter_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,10 @@ def get_data_permalink_slack(ztf_id):
'''
cutout = get_cutout(ztf_id)
curve = get_curve(ztf_id)
slack_client = WebClient(os.environ['ANOMALY_SLACK_TOKEN'])
if 'ANOMALY_SLACK_TOKEN' in os.environ:
slack_client = WebClient(os.environ['ANOMALY_SLACK_TOKEN'])
else:
raise KeyError("You need to set up ANOMALY_SLACK_TOKEN in your .bashrc")
try:
curve.seek(0)
cutout.seek(0)
Expand Down
Empty file added fink_filters/filter_known_tde/__init__.py
View file
Empty file.
56 changes: 56 additions & 0 deletions fink_filters/filter_known_tde/data/TDElist_Gezari2021.tsv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
NGC 5905 ROSAT X 0.01124 40.94 5.84 Bade et al. 1996
RX J1624+75 ROSAT X 0.0636 43.38 06.05 Grupe et al. 1999
RX J1242-11A ROSAT X 0.050 42.60 5.84 Komossa & Greiner 1999
RX J1420+53 ROSAT X 0.147 43.38 5.67 Greiner et al. 2000
GALEX D3-13 GALEX U 0.3698 43.98 4.66 Gezari et al. 2006
SDSS J1323+48 XMM X 0.0875 44.30 5.91 Esquej et al. 2007
GALEX D1-9 GALEX U 0.326 43.48 4.59 Gezari et al. 2008
TDXF J1347-32 ROSAT X 0.0366 42.73 6.14 Cappelluti et al. 2009
GALEX D23H-1 GALEX U 0.1855 43.95 4.70 Gezari et al. 2009
SDSS J1311-01 Chandra X 0.195 41.74 6.14 Maksym, Ulmer & Eracleous 2010
SwiftJ1644 Swift G 0.353 ND ND Bloom et al. 2011
2XMMi J1847-63 XMM X 0.0353 42.82 5.96 Lin et al. 2011
SDSS-TDE1 SDSS O 0.136 43.64 4.42 van Velzen et al. 2011
SDSS-TDE2 SDSS O 0.2515 44.54 4.37 van Velzen et al. 2011
PS1-10jh PS O 0.1696 44.47 4.59 Gezari et al. 2012
SDSS J1201+30 XMM X 0.146 45.00 06.06 Saxton et al. 2012
SwiftJ2058 Swift G 1.186 ND ND Cenko et al. 2012b
WINGS J1348+26 Chandra X 0.0651 41.79 06.06 Maksym et al. 2013
PS1-11af PS O 0.4046 44.16 4.28 Chornock et al. 2014
RBS 1032 ROSAT X 0.026 41.70 6.11 Maksym et al. 2014b
PTF-09ge PTF O 0.064 44.04 04.08 Arcavi et al. 2014
PTF-09axc PTF O 0.1146 43.46 04.08 Arcavi et al. 2014
PTF-09djl PTF O 0.184 44.42 4.41 Arcavi et al. 2014
ASASSN-14ae ASAS-SN O 0.0436 43.87 4.29 Holoien et al. 2014
3XMM J1521+07 XMM X 0.179 43.51 6.30 Lin et al. 2015
SwiftJ1112 Swift G 0.89 ND ND Brown et al. 2015
ASASSN-14li ASAS-SN O 0.02058 43.66 4.52 Holoien et al. 2016b
ASASSN-15lh ASAS-SN O 0.2326 45.34 4.30 Dong et al. 2016
ASASSN-15oi ASAS-SN O 0.0484 44.45 4.60 Holoien et al. 2016b
iPTF-16axa PTF O 0.108 43.82 4.46 Hung et al. 2017
iPTF-16fnl PTF O 0.0163 43.18 4.47 Blagorodnova et al. 2017
3XMM J1500+01 XMM X 0.1454 43.08 06.06 Lin et al. 2017
OGLE16aaa OGLE O 0.1655 44.22 4.36 Wyrzykowski et al. 2017
XMMSL1 J0740-85 XMM X 0.0173 42.61 6.00 Saxton et al. 2017
iPTF-15af PTF O 0.07897 44.10 4.85 Blagorodnova et al. 2019
AT2017eqx/PS17dhz PS O 0.1089 43.82 4.32 Nicholl et al. 2019
AT2018zr/PS18kh PS O 0.071 43.78 4.14 van Velzen et al. 2021
AT2018bsi/ZTF18aahqkbt ZTF O 0.051 43.87 4.53 van Velzen et al. 2021
AT2018dyb/ASASSN-18pg ASAS-SN O 0.018 44.08 4.40 Leloudas et al. 2019
AT2018fyk/ASASSN-18ul ASAS-SN O 0.059 44.48 4.54 Wevers et al. 2019
AT2018hco/ATLAS18way ATLAS O 0.088 44.25 4.39 van Velzen et al. 2021
AT2018hyz/ASASSN-18zj ASAS-SN O 0.04573 44.10 4.25 Gomez et al. 2020
AT2018iih/ATLAS18yzs ATLAS O 0.212 44.62 4.23 van Velzen et al. 2021
AT2018lna/ZTF19aabbnzo ZTF O 0.091 44.56 4.59 van Velzen et al. 2021
AT2018lni/ZTF18actaqdw ZTF O 0.138 44.21 4.38 van Velzen et al. 2021
AT2019ahk/ASASSN-19bt ASAS-SN O 0.0262 44.08 4.30 Holoien et al. 2019b
AT2019azh/ASASSN-19dj ASAS-SN O 0.0222 44.50 4.51 van Velzen et al. 2021
AT2019bhf/ZTF19aakswrb ZTF O 0.1206 43.91 4.27 van Velzen et al. 2021
AT2019cho/ZTF19aakiwze ZTF O 0.193 43.98 4.19 van Velzen et al. 2021
AT2019dsg/ZTF19aapreis ZTF O 0.0512 44.26 4.59 van Velzen et al. 2021
AT2019ehz/Gaia19bpt Gaia O 0.074 44.03 4.34 van Velzen et al. 2021
AT2019eve/ZTF19aatylnl ZTF O 0.0813 43.14 04.06 van Velzen et al. 2021
AT2019lwu/ZTF19abidbya ZTF O 0.117 43.60 4.14 van Velzen et al. 2021
AT2019meg/ZTF19abhhjcc ZTF O 0.152 44.36 4.44 van Velzen et al. 2021
AT2019mha/ATLAS19qqu ATLAS O 0.148 44.05 4.35 van Velzen et al. 2021
AT2019qiz/ZTF19abzrhgq ZTF O 0.0151 43.44 4.27 van Velzen et al. 2021
65 changes: 65 additions & 0 deletions fink_filters/filter_known_tde/data/Table1_Hammerstein
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
Title: The Final Season Reimagined: 30 Tidal Disruption Events from the
ZTF-I Survey
Authors: Hammerstein E., van Velzen S., Gezari S., Cenko S.B., Yao Y., Ward C.,
Frederick S., Villanueva N., Somalwar J.J., Graham M.J., Kulkarni S.R.,
Stern D., Andreoni I., Bellm E.C., Dekany R., Dhawan S., Drake A.J.,
Fremling C., Gatkine P., Groom S.L., Ho A.Y.Q., Kasliwal M.M.,
Karambelkar V., Kool E.C., Masci F.J., Medford M.S., Perley D.A.,
Purdum J., van Roestel J., Sharma Y., Sollerman J., Taggart K., Yan L.
Table: ZTF-I TDEs
================================================================================
Byte-by-byte Description of file: apjaca283t1_mrt.txt
--------------------------------------------------------------------------------
Bytes Format Units Label Explanations
--------------------------------------------------------------------------------
1- 9 A9 --- IAU IAU name
11- 22 A12 --- ZTF ZTF Name
24- 35 A12 --- Disc Discovery Name
37- 48 A12 --- GoT ZTF TDE Working Group nickname
50- 79 A30 --- ONames Other name(s)
81- 99 A19 --- Class First TDE classification (1)
101-115 A15 --- Spec TDE spectral class, Section 2.3
117-124 A8 --- zspec Spectroscopic redshift (2)
--------------------------------------------------------------------------------
Note (1): First TDE classification report, corresponding to either this
paper, to van Velzen et al. (2021) [2021ApJ...908....4V] or to the
following abbreviations. For AT2019azh see Hinkle et al. (2021)
[2021MNRAS.500.1673H] in addition to ATel #12568 --
ATel = Astronomer’s Telegram, https://astronomerstelegram.org/);
AN = AstroNotes, https://www.wis-tns.org/astronotes;
TNSCR = TNS classification reports.
Note (2): Redshifts were determined using host galaxy stellar absorption
lines or narrow emission lines associated with star formation, namely Ca
II H and K or narrow H{alpha} emission.
--------------------------------------------------------------------------------
AT2018zr ZTF18aabtxvd PS18kh Ned ATel #11444 TDE-H 0.075
AT2018bsi ZTF18aahqkbt ZTF18aahqkbt Jon ATel #12035 TDE-H+He 0.051
AT2018hco ZTF18abxftqm ATLAS18way Sansa ATel #12263 TDE-H 0.088
AT2018iih ZTF18acaqdaa ATLAS18yzs Jorah Gaia18dpo 2021ApJ...908....4V TDE-He 0.212
AT2018hyz ZTF18acpdvos ASASSN-18zj Gendry ATLAS18bafs ATel #12198 TDE-H+He 0.046
AT2018lni ZTF18actaqdw ZTF18actaqdw Arya 2021ApJ...908....4V TDE-H+He 0.138
AT2018lna ZTF19aabbnzo ZTF19aabbnzo Cersei ATel #12509 TDE-H+He 0.091
AT2018jbv ZTF18acnbpmd ZTF18acnbpmd Samwell ATLAS19acl, PS19aoz This paper TDE-featureless 0.340
AT2019cho ZTF19aakiwze ZTF19aakiwze Petyr 2021ApJ...908....4V TDE-H+He 0.193
AT2019bhf ZTF19aakswrb ZTF19aakswrb Varys 2021ApJ...908....4V TDE-H+He 0.121
AT2019azh ZTF17aaazdba ASASSN-19dj Jaime Gaia19bvo ATel #12568 * TDE-H+He 0.022
AT2019dsg ZTF19aapreis ZTF19aapreis Bran ATLAS19kl ATel #12752 TDE-H+He 0.051
AT2019ehz ZTF19aarioci Gaia19bpt Brienne ATel #12789 TDE-H 0.074
AT2019mha ZTF19abhejal ATLAS19qqu Bronn 2021ApJ...908....4V TDE-H+He 0.148
AT2019meg ZTF19abhhjcc ZTF19abhhjcc Margaery Gaia19dhd AN-2019-88 TDE-H 0.152
AT2019lwu ZTF19abidbya ZTF19abidbya Robb ATLAS19rnz, PS19ega 2021ApJ...908....4V TDE-H 0.117
AT2019qiz ZTF19abzrhgq ZTF19abzrhgq Melisandre ATLAS19vfr, Gaia19eks, PS19gdd ATel #13131 TDE-H+He 0.015
AT2019teq ZTF19accmaxo ZTF19accmaxo Missandei TNSCR #7482 TDE-H+He 0.087
AT2020pj ZTF20aabqihu ZTF20aabqihu Gilly ATLAS20cab TNSCR #7481 TDE-H+He 0.068
AT2019vcb ZTF19acspeuw ZTF19acspeuw Tormund Gaia19feb, ATLAS19bcyz TNSCR #7078 TDE-H+He 0.088
AT2020ddv ZTF20aamqmfk ATLAS20gee Shae ATel #13655 TDE-He 0.160
AT2020ocn ZTF18aakelin ZTF18aakelin Podrick ATel #13859 TDE-He 0.070
AT2020opy ZTF20abjwvae ZTF20abjwvae High Sparrow PS20fxm ATel #13944 TDE-H+He 0.159
AT2020mot ZTF20abfcszi ZTF20abfcszi Pycelle Gaia20ead ATel #13944 TDE-H+He 0.070
AT2020mbq ZTF20abefeab ZTF20abefeab Yara ATLAS20pfz, PS20grv This paper TDE-H 0.093
AT2020qhs ZTF20abowque ZTF20abowque Loras ATLAS20upw, PS20krl This paper TDE-featureless 0.345
AT2020riz ZTF20abrnwfc ZTF20abrnwfc Talisa PS20jop This paper TDE-featureless 0.435
AT2020wey ZTF20acitpfz ZTF20acitpfz Roose ATLAS20belb, Gaia20fck TNSCR #7769 TDE-H+He 0.027
AT2020zso ZTF20acqoiyt ZTF20acqoiyt Hodor ATLAS20bfok TNSCR #8025 TDE-H+He 0.057
AT2020ysg ZTF20abnorit ZTF20abnorit Osha ATLAS20bjqp, PS21cru This paper TDE-featureless 0.277\nl

2 changes: 2 additions & 0 deletions fink_filters/filter_known_tde/data/log.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Not found: ['RX J1624+75', 'RX J1242-11A', 'RX J1420+53', 'GALEX D3-13', 'SDSS J1323+48', 'GALEX D1-9', 'TDXF J1347-32', 'GALEX D23H-1', 'SDSS J1311-01', 'SwiftJ1644', '2XMMi J1847-63', 'SDSS-TDE1', 'SDSS-TDE2', 'SDSS J1201+30', 'SwiftJ2058', 'WINGS J1348+26', 'PTF-09ge', 'PTF-09axc', 'PTF-09djl', '3XMM J1521+07', 'SwiftJ1112', 'iPTF-16axa', 'iPTF-16fnl', '3XMM J1500+01', 'XMMSL1 J0740-85', 'iPTF-15af']
Number of TDE found: 44/70
61 changes: 61 additions & 0 deletions fink_filters/filter_known_tde/data/resolver.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import requests
import pandas as pd

APIURL = 'https://fink-portal.org'

data = {'name': [], 'ra': [], 'dec': []}

# Hammerstein TDEs
TDEs_hammerstein = pd.read_fwf('Table1_Hammerstein', skiprows=34, header=None)
tns_names = TDEs_hammerstein[0].to_list()

unknown = []
for name in tns_names:
r = requests.post(
'{}/api/v1/resolver'.format(APIURL),
json={'resolver': 'simbad', 'name': name}
)
if r.json() == []:
r = requests.post(
'{}/api/v1/resolver'.format(APIURL),
json={'resolver': 'tns', 'name': name}
)

if r.json() == []:
unknown.append(name)
else:
data['name'].append(r.json()[0]['oname'])
data['ra'].append(r.json()[0]['jradeg'])
data['dec'].append(r.json()[0]['jdedeg'])
# print(name, r.json())

# Gezari TDEs
pdf = pd.read_csv('TDElist_Gezari2021.tsv', sep='\t', header=None)
names = pdf[0].values

for name in names:
if '/' in name:
name = name.split('/')[0]
r = requests.post(
'{}/api/v1/resolver'.format(APIURL),
json={'resolver': 'simbad', 'name': name}
)
if r.json() == []:
r = requests.post(
'{}/api/v1/resolver'.format(APIURL),
json={'resolver': 'tns', 'name': name}
)

if r.json() == []:
unknown.append(name)
else:
data['name'].append(r.json()[0]['oname'])
data['ra'].append(r.json()[0]['jradeg'])
data['dec'].append(r.json()[0]['jdedeg'])
# print(name, r.json())
print('Not found: {}'.format(unknown))

pdf_final = pd.DataFrame.from_dict(data)
pdf_final = pdf_final.drop_duplicates()
print('Number of TDE found: {}/{}'.format(len(pdf_final), len(pdf_final) + len(unknown)))
pdf_final.to_parquet('tde.parquet')
Binary file added fink_filters/filter_known_tde/data/tde.parquet
View file
Binary file not shown.
144 changes: 144 additions & 0 deletions fink_filters/filter_known_tde/filter.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
# Copyright 2023 AstroLab Software
# Author: Julien Peloton
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pyspark.sql.functions import pandas_udf, PandasUDFType
from pyspark.sql.types import BooleanType

from astropy.coordinates import SkyCoord
from astropy import units as u

from fink_science.xmatch.utils import cross_match_astropy

from fink_filters.filter_anomaly_notification.filter_utils import msg_handler_slack
from fink_filters.tester import spark_unit_tests

import pandas as pd
import numpy as np
import os

def known_tde_(objectId, ra, dec, radius_arcsec=pd.Series([5])) -> pd.Series:
""" Return alerts matching with known TDEs
Parameters
----------
objectId: Pandas series
Colujmn containing ZTF object ID
ra: Pandas series
Column containing the RA values of alerts
dec: Pandas series
Column containing the DEC values of alerts
radius_arcsec: series
Radius for crossmatch, in arcsecond
Returns
----------
out: pandas.Series of bool
Return a Pandas DataFrame with the appropriate flag:
false for bad alert, and true for good alert.
Examples
----------
>>> pdf = pd.read_parquet('datatest_tde')
>>> classification = known_tde_(
... pdf['objectId'],
... pdf['candidate'].apply(lambda x: x['ra']),
... pdf['candidate'].apply(lambda x: x['dec']))
>>> print(np.sum(classification))
1
"""
curdir = os.path.dirname(os.path.abspath(__file__))
tdes = pd.read_parquet(curdir + '/data/tde.parquet')

catalog_tde = SkyCoord(
ra=np.array(tdes.ra, dtype=float) * u.degree,
dec=np.array(tdes.dec, dtype=float) * u.degree
)

pdf = pd.DataFrame(
{
'ra': ra,
'dec': dec,
'candid': range(len(ra)),
'objectId': objectId
}
)

catalog_ztf = SkyCoord(
ra=np.array(ra.values, dtype=float) * u.degree,
dec=np.array(dec.values, dtype=float) * u.degree
)

pdf_merge, mask, idx2 = cross_match_astropy(
pdf, catalog_ztf, catalog_tde, radius_arcsec=radius_arcsec
)

pdf_merge['match'] = False
pdf_merge.loc[mask, 'match'] = True

pdf_merge['intname'] = 'Unknown'
pdf_merge.loc[mask, 'intname'] = [
str(i).strip() for i in tdes['name'].astype(str).values[idx2]
]

if ('ANOMALY_SLACK_TOKEN' in os.environ) and ('GITHUB_ENV' not in os.environ):
# send to Slack recursively
for _, row in pdf_merge[mask].iterrows():
slack_data = []
t1 = f'<https://fink-portal.org/{row.objectId}|{row.objectId}> associated with {row.intname}'
slack_data.append(f'''{t1}''')

msg_handler_slack(slack_data, "known_tde_follow_up", init_msg='New TDE association!')

return pdf_merge['match']


@pandas_udf(BooleanType(), PandasUDFType.SCALAR)
def known_tde(objectId, ra, dec) -> pd.Series:
""" Pandas UDF for early_sn_candidates_
Parameters
----------
objectId: Pandas series
Column containing ZTF object ID
ra: Pandas series
Column containing the RA values of alerts
dec: Pandas series
Column containing the DEC values of alerts
Returns
----------
out: pandas.Series of bool
Return a Pandas DataFrame with the appropriate flag:
false for bad alert, and true for good alert.
Examples
----------
>>> from fink_utils.spark.utils import apply_user_defined_filter
>>> df = spark.read.format('parquet').load('datatest_tde')
>>> f = 'fink_filters.filter_known_tde.filter.known_tde'
>>> df = apply_user_defined_filter(df, f)
>>> print(df.count())
1
"""
series = known_tde_(objectId, ra, dec)
return series


if __name__ == "__main__":
""" Execute the test suite """

# Run the test suite
globs = globals()
spark_unit_tests(globs)

0 comments on commit 22d1ac8

Please sign in to comment.