From 8c126e19d1cd391e75f1ea2b72b8f993149a2641 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Fri, 31 May 2024 18:45:43 +0300
Subject: [PATCH 01/37] anomaly base

---
 .../filter_anomaly_notification/filter.py     | 41 +++++++++++++------
 .../filter_utils.py                           | 41 ++++++++++++++++++-
 requirements.txt                              |  2 +-
 3 files changed, 68 insertions(+), 16 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index 28adc11..c3595c3 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -24,11 +24,13 @@
 from fink_filters.tester import spark_unit_tests
 
 
+MODEL_POSTFIX = ['', '_alpha', '_beta', '_gamma']
+
 def anomaly_notification_(
         df_proc, threshold=10,
         send_to_tg=False, channel_id=None,
         send_to_slack=False, channel_name=None,
-        trick_par=10, cut_coords=False, history_period=90):
+        trick_par=10, cut_coords=False, history_period=90, model=''):
     """ Create event notifications with a high `anomaly_score` value
 
     Notes
@@ -68,6 +70,7 @@ def anomaly_notification_(
     history_period: int
             Time period in days for which the number
             of references is calculated
+    model: TODO
 
     Returns
     ----------
@@ -134,19 +137,21 @@ def anomaly_notification_(
             return pd.DataFrame()
 
     # Compute the median for the night
-    med = df_proc.select('anomaly_score').approxQuantile('anomaly_score', [0.5], 0.05)
+    buf_df = df_proc.select(f'anomaly_score{model}')
+    med = buf_df.approxQuantile(f'anomaly_score{model}', [0.5], 0.05)
     med = round(med[0], 2)
 
     # Extract anomalous objects
 
-    pdf_anomalies_ext = df_proc.sort(['anomaly_score'], ascending=True).limit(trick_par * threshold).toPandas()
+    pdf_anomalies_ext = df_proc.sort([f'anomaly_score{model}'], ascending=True).limit(trick_par * threshold).toPandas()
     pdf_anomalies_ext = pdf_anomalies_ext.drop_duplicates(['objectId'])
-    upper_bound = np.max(pdf_anomalies_ext['anomaly_score'].values[:threshold])
-    pdf_anomalies = pdf_anomalies_ext[pdf_anomalies_ext['anomaly_score'] <= upper_bound]
+    upper_bound = np.max(pdf_anomalies_ext[f'anomaly_score{model}'].values[:threshold])
+    pdf_anomalies = pdf_anomalies_ext[pdf_anomalies_ext[f'anomaly_score{model}'] <= upper_bound]
 
     history_objects = filter_utils.get_an_history(history_period)
 
-    tg_data, slack_data = [], []
+    tg_data, slack_data, base_data = [], [], []
+
     for _, row in pdf_anomalies.iterrows():
         gal = SkyCoord(ra=row.ra * u.degree, dec=row.dec * u.degree, frame='icrs').galactic
         oid = filter_utils.get_OID(row.ra, row.dec)
@@ -169,13 +174,14 @@ def anomaly_notification_(
         cutout.seek(0)
         cutout_perml = f"<{cutout_perml}|{' '}>"
         curve_perml = f"<{curve_perml}|{' '}>"
-        tg_data.append((f'''{t1a}
+        if model == '':
+            tg_data.append((f'''{t1a}
 {t_oid_1a}
 {t2_}
 {t3_}
 {t4_}
 {t5_}''', cutout, curve))
-        slack_data.append(f'''==========================
+            slack_data.append(f'''==========================
 {t1b}
 {t_oid_1b}
 {t2_}
@@ -183,19 +189,28 @@ def anomaly_notification_(
 {t4_}
 {t5_}
 {cutout_perml}{curve_perml}''')
+        base_data.append((row.objectId, f'''{t1b}
+{t_oid_1b}
+{t2_}
+{t3_}
+{t4_}
+{t5_}'''.replace('\n', '  \n'), cutout, curve))
+
     init_msg = f'Median anomaly score overnight: {med}.'
-    if cut_coords:
+    if cut_coords and model == '':
         init_msg += f'''
 (of the objects in the sky area)
 Sky area:
-    1) delta <= 20°
-    2) alpha ∈ (160°, 240°)
+1) delta <= 20°
+2) alpha ∈ (160°, 240°)
 Total number of objects per night in the area: {cut_count}.
 '''
-    if send_to_slack:
+    if send_to_slack and model == '':
         filter_utils.msg_handler_slack(slack_data, channel_name, init_msg)
-    if send_to_tg:
+    if send_to_tg and model == '':
         filter_utils.msg_handler_tg(tg_data, channel_id, init_msg)
+    if model != '':
+        filter_utils.load_to_anomaly_base(base_data, model)
     return pdf_anomalies
 
 
diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index af64b31..749baa1 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -20,7 +20,8 @@
 from collections import Counter
 import pandas as pd
 import numpy as np
-
+import json
+import markdown
 
 import matplotlib.pyplot as plt
 
@@ -141,7 +142,7 @@ def status_check(res):
             method,
             data={
                 "chat_id": "@fink_test",
-                "text": str(res.status_code)
+                "text": f'Error: {str(res.status_code)}, description: {res.text}'
             },
             timeout=25
         )
@@ -196,6 +197,14 @@ def msg_handler_slack(slack_data, channel_name, init_msg):
                 timeout=25
             )
 
+
+def get_user_tgid(receiver):
+    result = requests.get(f'http://157.136.253.53:24000/user/{receiver}')
+    if status_check(result):
+        return result.text.strip('"')
+    return None
+
+
 def msg_handler_tg(tg_data, channel_id, init_msg):
     '''
     Notes
@@ -263,6 +272,34 @@ def msg_handler_tg(tg_data, channel_id, init_msg):
         status_check(res)
         time.sleep(10)
 
+def load_to_anomaly_base(data, login):
+    res = requests.post('https://anomaly.fink-portal.org:443/user/signin', data={
+        'username': login,
+        'password': os.environ['ANOMALY_TG_TOKEN']
+    })
+    if status_check(res):
+        for ztf_id, text_data, cutout, curve in data:
+            cutout.seek(0)
+            curve.seek(0)
+            files = {
+                "image1": cutout,
+                "image2": curve
+            }
+            data = {
+                "description": text_data
+            }
+            params = {
+                "ztf_id": ztf_id
+            }
+            headers = {
+                "Authorization": f"Bearer {json.loads(res.text)['access_token']}"
+            }
+
+            response = requests.post('https://anomaly.fink-portal.org:443/images/upload', files=files, params=params, data=data,
+                                     headers=headers)
+            status_check(response)
+
+
 
 def get_OID(ra, dec):
     '''
diff --git a/requirements.txt b/requirements.txt
index 4023ac9..ad83ab4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,5 +14,5 @@ astroquery
 
 pandas
 pyarrow
-
+markdown
 h5py

From c4772c0b0d6dfaac2596f6a8d71e2518383c1eec Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sat, 1 Jun 2024 10:24:19 +0300
Subject: [PATCH 02/37] login fix

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 749baa1..661454c 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -274,7 +274,7 @@ def msg_handler_tg(tg_data, channel_id, init_msg):
 
 def load_to_anomaly_base(data, login):
     res = requests.post('https://anomaly.fink-portal.org:443/user/signin', data={
-        'username': login,
+        'username': login[1:],
         'password': os.environ['ANOMALY_TG_TOKEN']
     })
     if status_check(res):

From 354fb6fa607a44239eb0760635f849da619e25e4 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sat, 1 Jun 2024 16:16:00 +0300
Subject: [PATCH 03/37] small fixes

---
 fink_filters/filter_anomaly_notification/filter.py       | 7 +++----
 fink_filters/filter_anomaly_notification/filter_utils.py | 1 -
 requirements.txt                                         | 1 -
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index c3595c3..90f486a 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -24,7 +24,6 @@
 from fink_filters.tester import spark_unit_tests
 
 
-MODEL_POSTFIX = ['', '_alpha', '_beta', '_gamma']
 
 def anomaly_notification_(
         df_proc, threshold=10,
@@ -146,7 +145,7 @@ def anomaly_notification_(
     pdf_anomalies_ext = df_proc.sort([f'anomaly_score{model}'], ascending=True).limit(trick_par * threshold).toPandas()
     pdf_anomalies_ext = pdf_anomalies_ext.drop_duplicates(['objectId'])
     upper_bound = np.max(pdf_anomalies_ext[f'anomaly_score{model}'].values[:threshold])
-    pdf_anomalies = pdf_anomalies_ext[pdf_anomalies_ext[f'anomaly_score{model}'] <= upper_bound]
+    pdf_anomalies = pdf_anomalies_ext[pdf_anomalies_ext[f'anomaly_score{model}'] <= upper_bound].head(10)
 
     history_objects = filter_utils.get_an_history(history_period)
 
@@ -189,8 +188,8 @@ def anomaly_notification_(
 {t4_}
 {t5_}
 {cutout_perml}{curve_perml}''')
-        base_data.append((row.objectId, f'''{t1b}
-{t_oid_1b}
+        base_data.append((row.objectId, f'''{t1a}
+{t_oid_1a}
 {t2_}
 {t3_}
 {t4_}
diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 661454c..583639f 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -21,7 +21,6 @@
 import pandas as pd
 import numpy as np
 import json
-import markdown
 
 import matplotlib.pyplot as plt
 
diff --git a/requirements.txt b/requirements.txt
index ad83ab4..c9933bb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,5 +14,4 @@ astroquery
 
 pandas
 pyarrow
-markdown
 h5py

From f270d38bb4479ec67e654b1720d0ae74f49d4a81 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sun, 2 Jun 2024 10:12:51 +0300
Subject: [PATCH 04/37] docstrings update

---
 .../filter_anomaly_notification/filter.py     | 23 ++++++++-----
 .../filter_utils.py                           | 33 +++++++++++--------
 2 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index 90f486a..659cef7 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -69,7 +69,10 @@ def anomaly_notification_(
     history_period: int
             Time period in days for which the number
             of references is calculated
-    model: TODO
+    model: str
+        Name of the model used.
+        Name must start with a ‘_’ and be ‘_{user_name}’,
+        where user_name is the user name of the model at https://anomaly.fink-portal.org/.
 
     Returns
     ----------
@@ -84,7 +87,7 @@ def anomaly_notification_(
     >>> from fink_science.anomaly_detection.processor import anomaly_score
 
     >>> df = spark.read.format('parquet').load('datatest/regular')
-
+    >>> MODELS = ['', '_gamma', '_delta', '_epsilon', '_theta', '_omega'] # '' corresponds to the model for a telegram channel
     >>> what = [
     ...     'jd', 'fid', 'magpsf', 'sigmapsf',
     ...     'magnr', 'sigmagnr', 'isdiffpos', 'distnr']
@@ -104,13 +107,15 @@ def anomaly_notification_(
     ...     'cdistnr', 'cmagnr', 'csigmagnr', 'cisdiffpos']
 
     >>> df = df.withColumn('lc_features', extract_features_ad(*ad_args))
-    >>> df = df.withColumn("anomaly_score", anomaly_score("lc_features"))
-
-    >>> df_proc = df.select(
-    ...     'objectId', 'candidate.ra',
-    ...     'candidate.dec', 'candidate.rb',
-    ...     'anomaly_score', 'timestamp')
-    >>> df_out = anomaly_notification_(df_proc)
+    >>> for model in MODELS:
+    ...     df = df.withColumn(f'anomaly_score{model}', anomaly_score("lc_features", F.lit(model)))
+
+    >>> for model in MODELS:
+    ...     df_proc = df.select(
+    ...         'objectId', 'candidate.ra',
+    ...         'candidate.dec', 'candidate.rb',
+    ...         f'anomaly_score{model}', 'timestamp')
+    ...     df_out = anomaly_notification_(df_proc, model=model)
 
     # Disable communication
     >>> pdf_anomalies = anomaly_notification_(df_proc, threshold=10,
diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 583639f..844d8da 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -55,8 +55,7 @@ def get_an_history(delta_date=90):
     if status_check(history_data):
         res_obj = Counter(pd.read_json(io.BytesIO(history_data.content))['i:objectId'].values)
         return res_obj
-    else:
-        return Counter()
+    return Counter()
 
 
 def get_data_permalink_slack(ztf_id):
@@ -148,6 +147,7 @@ def status_check(res):
         return False
     return True
 
+
 def msg_handler_slack(slack_data, channel_name, init_msg):
     '''
     Notes
@@ -197,13 +197,6 @@ def msg_handler_slack(slack_data, channel_name, init_msg):
             )
 
 
-def get_user_tgid(receiver):
-    result = requests.get(f'http://157.136.253.53:24000/user/{receiver}')
-    if status_check(result):
-        return result.text.strip('"')
-    return None
-
-
 def msg_handler_tg(tg_data, channel_id, init_msg):
     '''
     Notes
@@ -271,9 +264,24 @@ def msg_handler_tg(tg_data, channel_id, init_msg):
         status_check(res)
         time.sleep(10)
 
-def load_to_anomaly_base(data, login):
+def load_to_anomaly_base(data, model):
+    '''
+
+    Parameters
+    ----------
+    data: list
+        A list of tuples of 4 elements each: (ZTF identifier: str,
+        notification text: str, cutout: BytesIO, light curve: BytesIO)
+    model: str
+        Name of the model used.
+        Name must start with a ‘_’ and be ‘_{user_name}’,
+        where user_name is the user name of the model at https://anomaly.fink-portal.org/.
+    Returns
+    -------
+    NONE
+    '''
     res = requests.post('https://anomaly.fink-portal.org:443/user/signin', data={
-        'username': login[1:],
+        'username': model[1:],
         'password': os.environ['ANOMALY_TG_TOKEN']
     })
     if status_check(res):
@@ -293,9 +301,8 @@ def load_to_anomaly_base(data, login):
             headers = {
                 "Authorization": f"Bearer {json.loads(res.text)['access_token']}"
             }
-
             response = requests.post('https://anomaly.fink-portal.org:443/images/upload', files=files, params=params, data=data,
-                                     headers=headers)
+                                     headers=headers, timeout=10)
             status_check(response)
 
 

From 228a3e7e82b39c3e09d6c78c7ff6d304e81175c2 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sun, 2 Jun 2024 10:32:32 +0300
Subject: [PATCH 05/37] flake8 fix

---
 fink_filters/filter_anomaly_notification/filter.py       | 1 -
 fink_filters/filter_anomaly_notification/filter_utils.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index 659cef7..4d8d8f1 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -24,7 +24,6 @@
 from fink_filters.tester import spark_unit_tests
 
 
-
 def anomaly_notification_(
         df_proc, threshold=10,
         send_to_tg=False, channel_id=None,
diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 844d8da..7b45170 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -306,7 +306,6 @@ def load_to_anomaly_base(data, model):
             status_check(response)
 
 
-
 def get_OID(ra, dec):
     '''
     Notes

From 6f6a26a878041339f78b1df0a2a82ea451c4b77b Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sun, 2 Jun 2024 10:43:39 +0300
Subject: [PATCH 06/37] docstring fix

---
 fink_filters/filter_anomaly_notification/filter.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index 4d8d8f1..04fce08 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -117,6 +117,10 @@ def anomaly_notification_(
     ...     df_out = anomaly_notification_(df_proc, model=model)
 
     # Disable communication
+    >>> df_proc = df.select(
+    ...         'objectId', 'candidate.ra',
+    ...         'candidate.dec', 'candidate.rb',
+    ...         'anomaly_score', 'timestamp')
     >>> pdf_anomalies = anomaly_notification_(df_proc, threshold=10,
     ...     send_to_tg=False, channel_id=None,
     ...     send_to_slack=False, channel_name=None)

From 328bae548c1b7ce5f70caaa78b61f25a227372dc Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sun, 2 Jun 2024 10:53:31 +0300
Subject: [PATCH 07/37] fix postfix row

---
 fink_filters/filter_anomaly_notification/filter.py       | 2 +-
 fink_filters/filter_anomaly_notification/filter_utils.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index 04fce08..0ff013d 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -172,7 +172,7 @@ def anomaly_notification_(
         t2_ += t_
         t3_ = f'UTC: {str(row.timestamp)[:-3]}'
         t4_ = f'Real bogus: {round(row.rb, 2)}'
-        t5_ = f'Anomaly score: {round(row.anomaly_score, 2)}'
+        t5_ = f'Anomaly score: {round(row[f"anomaly_score{model}"], 2)}'
         if row.objectId in history_objects:
             t5_ += f'''
 Detected as top-{threshold} in the last {history_period} days: {history_objects[row.objectId]} {'times' if history_objects[row.objectId] > 1 else 'time'}.'''
diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 7b45170..f3de78b 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -426,4 +426,5 @@ def get_curve(ztf_id):
     buf = io.BytesIO()
     plt.savefig(buf, format='png')
     buf.seek(0)
+    plt.close()
     return buf

From ad23482cebdf7ba1e375c20d0199796681b092dc Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sun, 2 Jun 2024 11:03:21 +0300
Subject: [PATCH 08/37] disable communication in doctest

---
 fink_filters/filter_anomaly_notification/filter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index 0ff013d..5faccbe 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -114,7 +114,7 @@ def anomaly_notification_(
     ...         'objectId', 'candidate.ra',
     ...         'candidate.dec', 'candidate.rb',
     ...         f'anomaly_score{model}', 'timestamp')
-    ...     df_out = anomaly_notification_(df_proc, model=model)
+    ...     df_out = anomaly_notification_(df_proc, send_to_tg=False, send_to_slack=False, model=model)
 
     # Disable communication
     >>> df_proc = df.select(

From d3aab61d2703532ee43beb714d2724a808553656 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sun, 2 Jun 2024 11:12:26 +0300
Subject: [PATCH 09/37] only beta model

---
 fink_filters/filter_anomaly_notification/filter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index 5faccbe..ebeae26 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -86,7 +86,7 @@ def anomaly_notification_(
     >>> from fink_science.anomaly_detection.processor import anomaly_score
 
     >>> df = spark.read.format('parquet').load('datatest/regular')
-    >>> MODELS = ['', '_gamma', '_delta', '_epsilon', '_theta', '_omega'] # '' corresponds to the model for a telegram channel
+    >>> MODELS = ['', '_beta'] # '' corresponds to the model for a telegram channel
     >>> what = [
     ...     'jd', 'fid', 'magpsf', 'sigmapsf',
     ...     'magnr', 'sigmagnr', 'isdiffpos', 'distnr']

From 422df79b7a144c828c503c46413a032045a910b3 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sun, 2 Jun 2024 15:28:06 +0300
Subject: [PATCH 10/37] bold font + new_param

---
 .../filter_anomaly_notification/filter.py     | 28 +++++++++++--------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index ebeae26..73fefdf 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -28,7 +28,7 @@ def anomaly_notification_(
         df_proc, threshold=10,
         send_to_tg=False, channel_id=None,
         send_to_slack=False, channel_name=None,
-        trick_par=10, cut_coords=False, history_period=90, model=''):
+        trick_par=10, cut_coords=False, history_period=90, send_to_anomaly_base=False,  model=''):
     """ Create event notifications with a high `anomaly_score` value
 
     Notes
@@ -72,6 +72,11 @@ def anomaly_notification_(
         Name of the model used.
         Name must start with a ‘_’ and be ‘_{user_name}’,
         where user_name is the user name of the model at https://anomaly.fink-portal.org/.
+    send_to_anomaly_base: bool
+        If True, notifications are uploaded to
+        https://anomaly.fink-portal.org/ in the selected model's
+        account. Only works for model != ‘’
+
 
     Returns
     ----------
@@ -114,7 +119,8 @@ def anomaly_notification_(
     ...         'objectId', 'candidate.ra',
     ...         'candidate.dec', 'candidate.rb',
     ...         f'anomaly_score{model}', 'timestamp')
-    ...     df_out = anomaly_notification_(df_proc, send_to_tg=False, send_to_slack=False, model=model)
+    ...     df_out = anomaly_notification_(df_proc, send_to_tg=False,
+    ...     send_to_slack=False, send_to_anomaly_base=True, model=model)
 
     # Disable communication
     >>> df_proc = df.select(
@@ -162,17 +168,17 @@ def anomaly_notification_(
     for _, row in pdf_anomalies.iterrows():
         gal = SkyCoord(ra=row.ra * u.degree, dec=row.dec * u.degree, frame='icrs').galactic
         oid = filter_utils.get_OID(row.ra, row.dec)
-        t1a = f'ID: [{row.objectId}](https://fink-portal.org/{row.objectId})'
+        t1a = f'*ID*: [{row.objectId}](https://fink-portal.org/{row.objectId})'
         t1b = f'ID: <https://fink-portal.org/{row.objectId}|{row.objectId}>'
-        t_oid_1a = f"DR OID (<1''): [{oid}](https://ztf.snad.space/view/{oid})"
+        t_oid_1a = f"*DR OID (<1'')*: [{oid}](https://ztf.snad.space/view/{oid})"
         t_oid_1b = f"DR OID (<1''): <https://ztf.snad.space/view/{oid}|{oid}>"
-        t2_ = f'GAL coordinates: {round(gal.l.deg, 6)},   {round(gal.b.deg, 6)}'
+        t2_ = f'*GAL coordinates*: {round(gal.l.deg, 6)},   {round(gal.b.deg, 6)}'
         t_ = f'''
-EQU: {row.ra},   {row.dec}'''
+*EQU*: {row.ra},   {row.dec}'''
         t2_ += t_
-        t3_ = f'UTC: {str(row.timestamp)[:-3]}'
-        t4_ = f'Real bogus: {round(row.rb, 2)}'
-        t5_ = f'Anomaly score: {round(row[f"anomaly_score{model}"], 2)}'
+        t3_ = f'*UTC*: {str(row.timestamp)[:-3]}'
+        t4_ = f'*Real bogus*: {round(row.rb, 2)}'
+        t5_ = f'*Anomaly score*: {round(row[f"anomaly_score{model}"], 2)}'
         if row.objectId in history_objects:
             t5_ += f'''
 Detected as top-{threshold} in the last {history_period} days: {history_objects[row.objectId]} {'times' if history_objects[row.objectId] > 1 else 'time'}.'''
@@ -212,9 +218,9 @@ def anomaly_notification_(
 2) alpha ∈ (160°, 240°)
 Total number of objects per night in the area: {cut_count}.
 '''
-    if send_to_slack and model == '':
+    if send_to_slack:
         filter_utils.msg_handler_slack(slack_data, channel_name, init_msg)
-    if send_to_tg and model == '':
+    if send_to_tg:
         filter_utils.msg_handler_tg(tg_data, channel_id, init_msg)
     if model != '':
         filter_utils.load_to_anomaly_base(base_data, model)

From 4f0024079b115097736c98e4757b18ab7a59d322 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sun, 2 Jun 2024 15:30:01 +0300
Subject: [PATCH 11/37] pep8...

---
 fink_filters/filter_anomaly_notification/filter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index 73fefdf..81312ec 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -28,7 +28,7 @@ def anomaly_notification_(
         df_proc, threshold=10,
         send_to_tg=False, channel_id=None,
         send_to_slack=False, channel_name=None,
-        trick_par=10, cut_coords=False, history_period=90, send_to_anomaly_base=False,  model=''):
+        trick_par=10, cut_coords=False, history_period=90, send_to_anomaly_base=False, model=''):
     """ Create event notifications with a high `anomaly_score` value
 
     Notes

From 80287edf98e1be22da6bf09364df9b115c07079c Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sun, 2 Jun 2024 15:42:30 +0300
Subject: [PATCH 12/37] new bold

---
 fink_filters/filter_anomaly_notification/filter.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index 81312ec..23a4cab 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -168,17 +168,17 @@ def anomaly_notification_(
     for _, row in pdf_anomalies.iterrows():
         gal = SkyCoord(ra=row.ra * u.degree, dec=row.dec * u.degree, frame='icrs').galactic
         oid = filter_utils.get_OID(row.ra, row.dec)
-        t1a = f'*ID*: [{row.objectId}](https://fink-portal.org/{row.objectId})'
+        t1a = f'**ID**: [{row.objectId}](https://fink-portal.org/{row.objectId})'
         t1b = f'ID: <https://fink-portal.org/{row.objectId}|{row.objectId}>'
-        t_oid_1a = f"*DR OID (<1'')*: [{oid}](https://ztf.snad.space/view/{oid})"
+        t_oid_1a = f"**DR OID (<1'')**: [{oid}](https://ztf.snad.space/view/{oid})"
         t_oid_1b = f"DR OID (<1''): <https://ztf.snad.space/view/{oid}|{oid}>"
-        t2_ = f'*GAL coordinates*: {round(gal.l.deg, 6)},   {round(gal.b.deg, 6)}'
+        t2_ = f'**GAL coordinates**: {round(gal.l.deg, 6)},   {round(gal.b.deg, 6)}'
         t_ = f'''
 *EQU*: {row.ra},   {row.dec}'''
         t2_ += t_
-        t3_ = f'*UTC*: {str(row.timestamp)[:-3]}'
-        t4_ = f'*Real bogus*: {round(row.rb, 2)}'
-        t5_ = f'*Anomaly score*: {round(row[f"anomaly_score{model}"], 2)}'
+        t3_ = f'**UTC**: {str(row.timestamp)[:-3]}'
+        t4_ = f'**Real bogus**: {round(row.rb, 2)}'
+        t5_ = f'**Anomaly score**: {round(row[f"anomaly_score{model}"], 2)}'
         if row.objectId in history_objects:
             t5_ += f'''
 Detected as top-{threshold} in the last {history_period} days: {history_objects[row.objectId]} {'times' if history_objects[row.objectId] > 1 else 'time'}.'''

From 902784ca85db9b708677683be68b409d93e3b530 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Sun, 2 Jun 2024 16:01:53 +0300
Subject: [PATCH 13/37] EQU bold

---
 fink_filters/filter_anomaly_notification/filter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index 23a4cab..a418d65 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -174,7 +174,7 @@ def anomaly_notification_(
         t_oid_1b = f"DR OID (<1''): <https://ztf.snad.space/view/{oid}|{oid}>"
         t2_ = f'**GAL coordinates**: {round(gal.l.deg, 6)},   {round(gal.b.deg, 6)}'
         t_ = f'''
-*EQU*: {row.ra},   {row.dec}'''
+**EQU**: {row.ra},   {row.dec}'''
         t2_ += t_
         t3_ = f'**UTC**: {str(row.timestamp)[:-3]}'
         t4_ = f'**Real bogus**: {round(row.rb, 2)}'

From 53d6b50afb7b460e51fd786c704576d7ecf6df6e Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Wed, 19 Jun 2024 01:26:10 +0300
Subject: [PATCH 14/37] Change basename for test

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index f3de78b..d63b192 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -280,7 +280,7 @@ def load_to_anomaly_base(data, model):
     -------
     NONE
     '''
-    res = requests.post('https://anomaly.fink-portal.org:443/user/signin', data={
+    res = requests.post('https://fink.matwey.name:443/user/signin', data={
         'username': model[1:],
         'password': os.environ['ANOMALY_TG_TOKEN']
     })

From 226aec60ac0febdfb427eab2b1d7af64b72abb30 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Wed, 19 Jun 2024 09:24:08 +0300
Subject: [PATCH 15/37] Change adr in rqst

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index d63b192..a349f2e 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -301,7 +301,7 @@ def load_to_anomaly_base(data, model):
             headers = {
                 "Authorization": f"Bearer {json.loads(res.text)['access_token']}"
             }
-            response = requests.post('https://anomaly.fink-portal.org:443/images/upload', files=files, params=params, data=data,
+            response = requests.post('https://fink.matwey.name:443/images/upload', files=files, params=params, data=data,
                                      headers=headers, timeout=10)
             status_check(response)
 

From 9e3c4083390bd7566049ba5b249a56f07a4b5bf1 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Tue, 25 Jun 2024 14:18:09 +0300
Subject: [PATCH 16/37] tg webhooks

---
 .../filter_utils.py                           | 89 +++++++++++++++++--
 1 file changed, 80 insertions(+), 9 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index f3de78b..b7f67b0 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -16,12 +16,14 @@
 import io
 import time
 import requests
+from functools import partial
 from datetime import datetime, timedelta
 from collections import Counter
 import pandas as pd
 import numpy as np
 import json
 
+
 import matplotlib.pyplot as plt
 
 from slack_sdk import WebClient
@@ -116,7 +118,7 @@ def get_data_permalink_slack(ztf_id):
     return cutout, curve, result['files'][0]['permalink'], result['files'][1]['permalink']
 
 
-def status_check(res):
+def status_check(res, source='not defined'):
     '''
     Checks whether the request was successful.
     In case of an error, sends information about the error to the @fink_test telegram channel
@@ -124,7 +126,7 @@ def status_check(res):
     Parameters
     ----------
     res : Response object
-
+    source : source of log
     Returns
     -------
         result : bool
@@ -140,7 +142,7 @@ def status_check(res):
             method,
             data={
                 "chat_id": "@fink_test",
-                "text": f'Error: {str(res.status_code)}, description: {res.text}'
+                "text": f'Source: {source}, error: {str(res.status_code)}, description: {res.text}'
             },
             timeout=25
         )
@@ -237,6 +239,14 @@ def msg_handler_tg(tg_data, channel_id, init_msg):
     )
     status_check(res)
     time.sleep(10)
+    inline_keyboard = {
+        "inline_keyboard": [
+            [
+                {"text": "Anomaly", "callback_data": "ANOMALY"},
+                {"text": "Not anomaly", "callback_data": "NOTANOMALY"}
+            ]
+        ]
+    }
     for text_data, cutout, curve in tg_data:
         res = requests.post(
             method,
@@ -253,7 +263,8 @@ def msg_handler_tg(tg_data, channel_id, init_msg):
                         "type" : "photo",
                         "media": "attach://first"
                     }}
-                ]'''
+                ]''',
+                "reply_markup": inline_keyboard
             },
             files={
                 "second": cutout,
@@ -280,11 +291,19 @@ def load_to_anomaly_base(data, model):
     -------
     NONE
     '''
-    res = requests.post('https://anomaly.fink-portal.org:443/user/signin', data={
-        'username': model[1:],
+    username = model[1:]
+    res = requests.post('https://fink.matwey.name:443/user/signin', data={
+        'username': username,
         'password': os.environ['ANOMALY_TG_TOKEN']
     })
-    if status_check(res):
+    if status_check(res, 'load_to_anomaly_base_login'):
+        # TODO: Загрузка tg_id из базы
+        tg_id_data = requests.get(url=f'https://fink.matwey.name:443/user/get_tgid/{username}')
+        if status_check(tg_id_data, 'tg_id loading'):
+            tg_id_data = tg_id_data.content.decode('utf-8')
+        else:
+            tg_id_data = 'ND'
+
         for ztf_id, text_data, cutout, curve in data:
             cutout.seek(0)
             curve.seek(0)
@@ -301,9 +320,61 @@ def load_to_anomaly_base(data, model):
             headers = {
                 "Authorization": f"Bearer {json.loads(res.text)['access_token']}"
             }
-            response = requests.post('https://anomaly.fink-portal.org:443/images/upload', files=files, params=params, data=data,
+            response = requests.post('https://fink.matwey.name:443/images/upload', files=files, params=params, data=data,
                                      headers=headers, timeout=10)
-            status_check(response)
+            status_check(response, 'upload to anomaly base')
+            cutout.seek(0)
+            curve.seek(0)
+            #send in tg personal
+            if tg_id_data == 'ND':
+                continue
+            inline_keyboard = {
+                "inline_keyboard": [
+                    [
+                        {"text": "Anomaly", "callback_data": f"A_{ztf_id}"},
+                        {"text": "Not anomaly", "callback_data": f"NA_{ztf_id}"}
+                    ]
+                ]
+            }
+
+            url = "https://api.telegram.org/bot"
+            url += os.environ['ANOMALY_TG_TOKEN']
+            method = url + "/sendMediaGroup"
+
+            res = requests.post(
+                method,
+                params={
+                    "chat_id": tg_id_data,
+                    "media": f'''[
+                            {{
+                                "type" : "photo",
+                                "media": "attach://second",
+                                "caption" : "{text_data}",
+                                "parse_mode": "markdown"
+                            }},
+                            {{
+                                "type" : "photo",
+                                "media": "attach://first"
+                            }}
+                        ]'''
+                },
+                files={
+                    "second": cutout,
+                    "first": curve,
+                },
+                timeout=25
+            )
+            if status_check(res, f'individual sending to {tg_id_data}'):
+                res = requests.post(
+                    f"https://api.telegram.org/bot{os.environ['ANOMALY_TG_TOKEN']}/sendMessage",
+                    json={
+                        "chat_id": tg_id_data,
+                        "text": f"Feedback for {ztf_id}:",
+                        "reply_markup": inline_keyboard
+                    },
+                    timeout=25
+                )
+                status_check(res, f'button individual sending to {tg_id_data}')
 
 
 def get_OID(ra, dec):

From 919d689517d0e63c695540ae823147e5ce7a9312 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Tue, 25 Jun 2024 14:29:21 +0300
Subject: [PATCH 17/37] flake8

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 9acd544..cfc897a 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -324,7 +324,7 @@ def load_to_anomaly_base(data, model):
             status_check(response, 'upload to anomaly base')
             cutout.seek(0)
             curve.seek(0)
-            #send in tg personal
+            # send in tg personal
             if tg_id_data == 'ND':
                 continue
             inline_keyboard = {

From 11ea4baa4ac1c96883fd8a9e96a6f84bdc735fc7 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Tue, 25 Jun 2024 20:53:42 +0300
Subject: [PATCH 18/37] add sleep

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index cfc897a..90f2481 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -291,6 +291,7 @@ def load_to_anomaly_base(data, model):
     NONE
     '''
     username = model[1:]
+    time.sleep(3)
     res = requests.post('https://fink.matwey.name:443/user/signin', data={
         'username': username,
         'password': os.environ['ANOMALY_TG_TOKEN']

From 9afde1a672d7595acb3c438992b4feba0e6d5bb7 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Tue, 25 Jun 2024 21:39:38 +0300
Subject: [PATCH 19/37] tg_id to int

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 90f2481..f695f75 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -328,6 +328,7 @@ def load_to_anomaly_base(data, model):
             # send in tg personal
             if tg_id_data == 'ND':
                 continue
+            tg_id_data = int(tg_id_data)
             inline_keyboard = {
                 "inline_keyboard": [
                     [

From ae5231650aa3ff9d72e5e651261068eca782d5b0 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Tue, 25 Jun 2024 22:23:18 +0300
Subject: [PATCH 20/37] fix " symbol + more logs

---
 .../filter_anomaly_notification/filter_utils.py    | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index f695f75..31000f8 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -53,7 +53,7 @@ def get_an_history(delta_date=90):
         }
     )
 
-    if status_check(history_data):
+    if status_check(history_data, 'checking history'):
         res_obj = Counter(pd.read_json(io.BytesIO(history_data.content))['i:objectId'].values)
         return res_obj
     return Counter()
@@ -236,7 +236,7 @@ def msg_handler_tg(tg_data, channel_id, init_msg):
         },
         timeout=25
     )
-    status_check(res)
+    status_check(res, 'sending to tg_channel (init)')
     time.sleep(10)
     inline_keyboard = {
         "inline_keyboard": [
@@ -271,7 +271,7 @@ def msg_handler_tg(tg_data, channel_id, init_msg):
             },
             timeout=25
         )
-        status_check(res)
+        status_check(res, 'sending to tg_channel (main messages)')
         time.sleep(10)
 
 def load_to_anomaly_base(data, model):
@@ -328,7 +328,7 @@ def load_to_anomaly_base(data, model):
             # send in tg personal
             if tg_id_data == 'ND':
                 continue
-            tg_id_data = int(tg_id_data)
+            tg_id_data = int(tg_id_data.replace('"', ''))
             inline_keyboard = {
                 "inline_keyboard": [
                     [
@@ -398,7 +398,7 @@ def get_OID(ra, dec):
     '''
     r = requests.get(
         url=f'http://db.ztf.snad.space/api/v3/data/latest/circle/full/json?ra={ra}&dec={dec}&radius_arcsec=1')
-    if not status_check(r):
+    if not status_check(r, 'get cross from snad'):
         return None
     oids = [key for key, _ in r.json().items()]
     if oids:
@@ -429,7 +429,7 @@ def get_cutout(ztf_id):
         },
         timeout=25
     )
-    status_check(r)
+    status_check(r, 'get cutouts')
     return io.BytesIO(r.content)
 
 def get_curve(ztf_id):
@@ -452,7 +452,7 @@ def get_curve(ztf_id):
             'withupperlim': 'True'
         }
     )
-    if not status_check(r):
+    if not status_check(r, 'getting curve'):
         return None
 
     # Format output in a DataFrame

From 0c37c9ffbeda4926d578a3b1d2877530605838a8 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Tue, 25 Jun 2024 22:53:53 +0300
Subject: [PATCH 21/37] access token fix

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 31000f8..4614f68 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -297,7 +297,7 @@ def load_to_anomaly_base(data, model):
         'password': os.environ['ANOMALY_TG_TOKEN']
     })
     if status_check(res, 'load_to_anomaly_base_login'):
-        # TODO: Загрузка tg_id из базы
+        access_token = json.loads(res.text)['access_token']
         tg_id_data = requests.get(url=f'https://fink.matwey.name:443/user/get_tgid/{username}')
         if status_check(tg_id_data, 'tg_id loading'):
             tg_id_data = tg_id_data.content.decode('utf-8')
@@ -318,7 +318,7 @@ def load_to_anomaly_base(data, model):
                 "ztf_id": ztf_id
             }
             headers = {
-                "Authorization": f"Bearer {json.loads(res.text)['access_token']}"
+                "Authorization": f"Bearer {access_token}"
             }
             response = requests.post('https://fink.matwey.name:443/images/upload', files=files, params=params, data=data,
                                      headers=headers, timeout=10)

From 00e9fabd479aca62a70fe3036b3d5ce181952b10 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Tue, 25 Jun 2024 23:15:52 +0300
Subject: [PATCH 22/37] fix tg_id to int multiple, timeout snad cross 5 sec

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 4614f68..b34b667 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -301,6 +301,7 @@ def load_to_anomaly_base(data, model):
         tg_id_data = requests.get(url=f'https://fink.matwey.name:443/user/get_tgid/{username}')
         if status_check(tg_id_data, 'tg_id loading'):
             tg_id_data = tg_id_data.content.decode('utf-8')
+            tg_id_data = int(tg_id_data.replace('"', ''))
         else:
             tg_id_data = 'ND'
 
@@ -328,7 +329,7 @@ def load_to_anomaly_base(data, model):
             # send in tg personal
             if tg_id_data == 'ND':
                 continue
-            tg_id_data = int(tg_id_data.replace('"', ''))
+
             inline_keyboard = {
                 "inline_keyboard": [
                     [
@@ -397,7 +398,9 @@ def get_OID(ra, dec):
             ZTF DR OID
     '''
     r = requests.get(
-        url=f'http://db.ztf.snad.space/api/v3/data/latest/circle/full/json?ra={ra}&dec={dec}&radius_arcsec=1')
+        url=f'http://db.ztf.snad.space/api/v3/data/latest/circle/full/json?ra={ra}&dec={dec}&radius_arcsec=1',
+        timeout=5
+    )
     if not status_check(r, 'get cross from snad'):
         return None
     oids = [key for key, _ in r.json().items()]

From 045c858595e47adf4f454f04d839be2c1d9c9a54 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Tue, 25 Jun 2024 23:22:47 +0300
Subject: [PATCH 23/37] fix timeout exception

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index b34b667..c2d5f91 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -397,9 +397,9 @@ def get_OID(ra, dec):
         out: str
             ZTF DR OID
     '''
+
     r = requests.get(
-        url=f'http://db.ztf.snad.space/api/v3/data/latest/circle/full/json?ra={ra}&dec={dec}&radius_arcsec=1',
-        timeout=5
+        url=f'http://db.ztf.snad.space/api/v3/data/latest/circle/full/json?ra={ra}&dec={dec}&radius_arcsec=1'
     )
     if not status_check(r, 'get cross from snad'):
         return None

From e8c5781617e6e5157000447e39798f9329a1d8fd Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <timofei.psheno@gmail.com>
Date: Tue, 25 Jun 2024 23:38:07 +0300
Subject: [PATCH 24/37] add try/except in OID

---
 .../filter_anomaly_notification/filter_utils.py        | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index c2d5f91..5301b86 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -397,10 +397,12 @@ def get_OID(ra, dec):
         out: str
             ZTF DR OID
     '''
-
-    r = requests.get(
-        url=f'http://db.ztf.snad.space/api/v3/data/latest/circle/full/json?ra={ra}&dec={dec}&radius_arcsec=1'
-    )
+    try:
+        r = requests.get(
+            url=f'http://db.ztf.snad.space/api/v3/data/latest/circle/full/json?ra={ra}&dec={dec}&radius_arcsec=1'
+        )
+    except:
+        return None
     if not status_check(r, 'get cross from snad'):
         return None
     oids = [key for key, _ in r.json().items()]

From 034f30eb5c7b6aa0ed27955a4e059cf7e3fd8f30 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Mon, 15 Jul 2024 13:49:57 +0300
Subject: [PATCH 25/37] pep8

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 5301b86..3c83a9a 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -401,7 +401,7 @@ def get_OID(ra, dec):
         r = requests.get(
             url=f'http://db.ztf.snad.space/api/v3/data/latest/circle/full/json?ra={ra}&dec={dec}&radius_arcsec=1'
         )
-    except:
+    except Exception:
         return None
     if not status_check(r, 'get cross from snad'):
         return None

From ee81694bcd39e17f028f7399ffa9e727a36b7e9f Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Sat, 27 Jul 2024 16:13:15 +0300
Subject: [PATCH 26/37] science template + new colormap

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 3c83a9a..fd869b8 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -430,7 +430,8 @@ def get_cutout(ztf_id):
         'https://fink-portal.org/api/v1/cutouts',
         json={
             'objectId': ztf_id,
-            'kind': 'Difference',
+            'kind': 'Science',
+            'colormap': 'PuBu_r'
         },
         timeout=25
     )

From 0bfde2adfdacc1760392d145aa4ddd6629a8135d Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Sat, 27 Jul 2024 16:20:32 +0300
Subject: [PATCH 27/37] fix

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index fd869b8..f0eb42f 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -430,8 +430,7 @@ def get_cutout(ztf_id):
         'https://fink-portal.org/api/v1/cutouts',
         json={
             'objectId': ztf_id,
-            'kind': 'Science',
-            'colormap': 'PuBu_r'
+            'kind': 'Science'
         },
         timeout=25
     )

From c8cf8b5286464c2309542f88b4214aa0ecc8040f Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Sat, 27 Jul 2024 17:20:09 +0300
Subject: [PATCH 28/37] new cutout test

---
 .../filter_anomaly_notification/filter_utils.py   | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index f0eb42f..a4454fe 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -427,15 +427,22 @@ def get_cutout(ztf_id):
 
     '''
     r = requests.post(
-        'https://fink-portal.org/api/v1/cutouts',
-        json={
+        'https://fink-portal.org/api/v1/objects',
+         json={
             'objectId': ztf_id,
-            'kind': 'Science'
+            'withcutouts': 'True'
         },
         timeout=25
     )
+    pdf = pd.read_json(io.BytesIO(r.content))
+    data = np.log(np.array(pdf['b:cutoutTemplate_stampData'].to_numpy()[0]))
+    plt.axis('off')
+    image_data = plt.imshow(data, cmap='PuBu_r')
     status_check(r, 'get cutouts')
-    return io.BytesIO(r.content)
+    buf = io.BytesIO()
+    plt.savefig(buf, format='png')
+    buf.seek(0)
+    return buf
 
 def get_curve(ztf_id):
     '''

From c47fccbeaa56ad88fe0badb585eab712933341e9 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Sat, 27 Jul 2024 17:35:02 +0300
Subject: [PATCH 29/37] add axis

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index a4454fe..6b6501d 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -434,14 +434,16 @@ def get_cutout(ztf_id):
         },
         timeout=25
     )
+    status_check(r, 'get cutouts')
+    fig, ax = plt.subplots()
     pdf = pd.read_json(io.BytesIO(r.content))
     data = np.log(np.array(pdf['b:cutoutTemplate_stampData'].to_numpy()[0]))
-    plt.axis('off')
-    image_data = plt.imshow(data, cmap='PuBu_r')
-    status_check(r, 'get cutouts')
+    ax.imshow(data, cmap='PuBu_r')
+    ax.axis('off')
     buf = io.BytesIO()
     plt.savefig(buf, format='png')
     buf.seek(0)
+    plt.close(fig)
     return buf
 
 def get_curve(ztf_id):

From 7064e0845e54ae7ae3f31e22953658a8dd373d25 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Sat, 27 Jul 2024 18:14:09 +0300
Subject: [PATCH 30/37] Template -> Scince

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 6b6501d..6bce1a0 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -437,7 +437,7 @@ def get_cutout(ztf_id):
     status_check(r, 'get cutouts')
     fig, ax = plt.subplots()
     pdf = pd.read_json(io.BytesIO(r.content))
-    data = np.log(np.array(pdf['b:cutoutTemplate_stampData'].to_numpy()[0]))
+    data = np.log(np.array(pdf['b:cutoutScience_stampData'].to_numpy()[0]))
     ax.imshow(data, cmap='PuBu_r')
     ax.axis('off')
     buf = io.BytesIO()

From 9edcf0f31d7cfe2f1457cf621e535eea0a4cabe6 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Sat, 27 Jul 2024 18:23:00 +0300
Subject: [PATCH 31/37] pep8 + exc

---
 .../filter_anomaly_notification/filter_utils.py      | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 6bce1a0..60b3c75 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -427,14 +427,12 @@ def get_cutout(ztf_id):
 
     '''
     r = requests.post(
-        'https://fink-portal.org/api/v1/objects',
-         json={
-            'objectId': ztf_id,
-            'withcutouts': 'True'
-        },
-        timeout=25
+        "https://fink-portal.org/api/v1/objects",
+        json={"objectId": ztf_id, "withcutouts": "True"},
+        timeout=25,
     )
-    status_check(r, 'get cutouts')
+    if not status_check(r, 'get cutouts'):
+        return io.BytesIO()
     fig, ax = plt.subplots()
     pdf = pd.read_json(io.BytesIO(r.content))
     data = np.log(np.array(pdf['b:cutoutScience_stampData'].to_numpy()[0]))

From 54e8b806c43a9b3c76c8540549946e01ed9e1a1f Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Sat, 27 Jul 2024 18:34:20 +0300
Subject: [PATCH 32/37] fix log

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 60b3c75..7018936 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -431,11 +431,13 @@ def get_cutout(ztf_id):
         json={"objectId": ztf_id, "withcutouts": "True"},
         timeout=25,
     )
-    if not status_check(r, 'get cutouts'):
+    if not status_check(r, 'get cutouts'): 
         return io.BytesIO()
     fig, ax = plt.subplots()
     pdf = pd.read_json(io.BytesIO(r.content))
-    data = np.log(np.array(pdf['b:cutoutScience_stampData'].to_numpy()[0]))
+    data = np.array(pdf['b:cutoutScience_stampData'].to_numpy()[0])
+    data[data <= 0] = 1e-10
+    data = np.log(data)
     ax.imshow(data, cmap='PuBu_r')
     ax.axis('off')
     buf = io.BytesIO()

From d74e0dc121f036d519903e8feef21049c77272dc Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Sat, 27 Jul 2024 18:44:32 +0300
Subject: [PATCH 33/37] data is None fix

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 7018936..44501d3 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -431,11 +431,13 @@ def get_cutout(ztf_id):
         json={"objectId": ztf_id, "withcutouts": "True"},
         timeout=25,
     )
-    if not status_check(r, 'get cutouts'): 
+    if not status_check(r, 'get cutouts'):
         return io.BytesIO()
     fig, ax = plt.subplots()
     pdf = pd.read_json(io.BytesIO(r.content))
     data = np.array(pdf['b:cutoutScience_stampData'].to_numpy()[0])
+    if data is None:
+        return return io.BytesIO()
     data[data <= 0] = 1e-10
     data = np.log(data)
     ax.imshow(data, cmap='PuBu_r')

From cfeaf64a05112b7be4b21f95742e72448832ff31 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Sat, 27 Jul 2024 18:45:45 +0300
Subject: [PATCH 34/37] fix

---
 fink_filters/filter_anomaly_notification/filter_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 44501d3..5a86fa5 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -437,7 +437,7 @@ def get_cutout(ztf_id):
     pdf = pd.read_json(io.BytesIO(r.content))
     data = np.array(pdf['b:cutoutScience_stampData'].to_numpy()[0])
     if data is None:
-        return return io.BytesIO()
+        return io.BytesIO()
     data[data <= 0] = 1e-10
     data = np.log(data)
     ax.imshow(data, cmap='PuBu_r')

From 21c8d80cce41af43f0b56836bd5fe654230bf9ae Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Sat, 27 Jul 2024 18:57:46 +0300
Subject: [PATCH 35/37] fix

---
 .../filter_utils.py                           | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 5a86fa5..5515bb9 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -436,16 +436,17 @@ def get_cutout(ztf_id):
     fig, ax = plt.subplots()
     pdf = pd.read_json(io.BytesIO(r.content))
     data = np.array(pdf['b:cutoutScience_stampData'].to_numpy()[0])
-    if data is None:
+    try:
+        data[data <= 0] = 1e-10
+        data = np.log(data)
+        ax.imshow(data, cmap='PuBu_r')
+        ax.axis('off')
+        buf = io.BytesIO()
+        plt.savefig(buf, format='png')
+        buf.seek(0)
+        plt.close(fig)
+    except TypeError:
         return io.BytesIO()
-    data[data <= 0] = 1e-10
-    data = np.log(data)
-    ax.imshow(data, cmap='PuBu_r')
-    ax.axis('off')
-    buf = io.BytesIO()
-    plt.savefig(buf, format='png')
-    buf.seek(0)
-    plt.close(fig)
     return buf
 
 def get_curve(ztf_id):

From 27598ed7639b43825c948f4a58dedc704f11de91 Mon Sep 17 00:00:00 2001
From: Timofei Pshenichnyy <93309519+Knispel2@users.noreply.github.com>
Date: Sat, 27 Jul 2024 19:05:43 +0300
Subject: [PATCH 36/37] remove

---
 .../filter_utils.py                           | 28 ++++++-------------
 1 file changed, 8 insertions(+), 20 deletions(-)

diff --git a/fink_filters/filter_anomaly_notification/filter_utils.py b/fink_filters/filter_anomaly_notification/filter_utils.py
index 5515bb9..f0eb42f 100755
--- a/fink_filters/filter_anomaly_notification/filter_utils.py
+++ b/fink_filters/filter_anomaly_notification/filter_utils.py
@@ -427,27 +427,15 @@ def get_cutout(ztf_id):
 
     '''
     r = requests.post(
-        "https://fink-portal.org/api/v1/objects",
-        json={"objectId": ztf_id, "withcutouts": "True"},
-        timeout=25,
+        'https://fink-portal.org/api/v1/cutouts',
+        json={
+            'objectId': ztf_id,
+            'kind': 'Science'
+        },
+        timeout=25
     )
-    if not status_check(r, 'get cutouts'):
-        return io.BytesIO()
-    fig, ax = plt.subplots()
-    pdf = pd.read_json(io.BytesIO(r.content))
-    data = np.array(pdf['b:cutoutScience_stampData'].to_numpy()[0])
-    try:
-        data[data <= 0] = 1e-10
-        data = np.log(data)
-        ax.imshow(data, cmap='PuBu_r')
-        ax.axis('off')
-        buf = io.BytesIO()
-        plt.savefig(buf, format='png')
-        buf.seek(0)
-        plt.close(fig)
-    except TypeError:
-        return io.BytesIO()
-    return buf
+    status_check(r, 'get cutouts')
+    return io.BytesIO(r.content)
 
 def get_curve(ztf_id):
     '''

From 6dc434d5b85a26a21b66d2be04bc681d641fb91e Mon Sep 17 00:00:00 2001
From: Julien <peloton@lal.in2p3.fr>
Date: Mon, 29 Jul 2024 10:17:01 +0200
Subject: [PATCH 37/37] Fix triple quote format

---
 fink_filters/filter_anomaly_notification/filter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fink_filters/filter_anomaly_notification/filter.py b/fink_filters/filter_anomaly_notification/filter.py
index c580028..2f97476 100755
--- a/fink_filters/filter_anomaly_notification/filter.py
+++ b/fink_filters/filter_anomaly_notification/filter.py
@@ -215,7 +215,7 @@ def anomaly_notification_(
 
     init_msg = f'Median anomaly score overnight: {med}.'
     if cut_coords and model == '':
-        init_msg += f'''
+        init_msg += f"""
 (of the objects in the sky area)
 Sky area:
 1) delta <= 20°