Crossmatch with SPICY

.github/workflows/run_test.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        container: ["julienpeloton/fink-ci:prod", "julienpeloton/fink-ci:dev"]
+        container: ["julienpeloton/fink-ci:dev"]
 
     container:
       image: ${{ matrix.container }}

fink_filters/classification.py

@@ -84,7 +84,7 @@ def extract_fink_classification_(
 
     Examples
     ---------
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = extract_fink_classification_(
     ...     pdf['cdsxmatch'],
     ...     pdf['roid'],
@@ -184,7 +184,7 @@ def extract_fink_classification_from_pdf(pdf):
     out: pandas.Series of string
         Return a Pandas series with the classification tag
 
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = extract_fink_classification_from_pdf(pdf)
     >>> pdf['class'] = classification
     >>> pdf.groupby('class').count().sort_values('objectId', ascending=False)['objectId'].head(10)

fink_filters/filter_anomaly_notification/filter.py

@@ -81,7 +81,7 @@ def anomaly_notification_(
     >>> from fink_science.ad_features.processor import extract_features_ad
     >>> from fink_science.anomaly_detection.processor import anomaly_score
 
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
 
     >>> what = [
     ...     'jd', 'fid', 'magpsf', 'sigmapsf',
@@ -114,9 +114,8 @@ def anomaly_notification_(
     >>> pdf_anomalies = anomaly_notification_(df_proc, threshold=10,
     ...     send_to_tg=False, channel_id=None,
     ...     send_to_slack=False, channel_name=None)
-    >>> print(pdf_anomalies['objectId'].values)
-    ['ZTF21acoshvy' 'ZTF18abgjtxx' 'ZTF19acevxhv' 'ZTF19aboujyi' 'ZTF18aapgymv'
-     'ZTF18abbtxsx' 'ZTF18aaakhsv' 'ZTF18aaypnnd' 'ZTF18aapoack' 'ZTF18abzvnya']
+    >>> print(sorted(pdf_anomalies['objectId'].values))
+    ['ZTF18aaakhsv', 'ZTF18aabeyfi', 'ZTF18aapgymv', 'ZTF18aapoack', 'ZTF18abbtxsx', 'ZTF18abgjtxx', 'ZTF18abzvnya', 'ZTF19aboujyi', 'ZTF19acevxhv', 'ZTF21acoshvy']
 
     # Check cut_coords
     >>> pdf_anomalies = anomaly_notification_(df_proc, threshold=10,

fink_filters/filter_blazar/filter.py

@@ -39,7 +39,7 @@ def blazar(cdsxmatch: Any) -> pd.Series:
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_blazar.filter.blazar'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_early_kn_candidates/filter.py

@@ -189,7 +189,7 @@ def early_kn_candidates_(
 
     Examples
     ----------
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = early_kn_candidates_(
     ...     pdf['candidate'].apply(lambda x: x['drb']),
     ...     pdf['candidate'].apply(lambda x: x['classtar']),
@@ -267,7 +267,7 @@ def early_kn_candidates(
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_early_kn_candidates.filter.early_kn_candidates'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_early_sn_candidates/filter.py

@@ -51,7 +51,7 @@ def early_sn_candidates_(
 
     Examples
     ----------
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = early_sn_candidates_(
     ...     pdf['cdsxmatch'],
     ...     pdf['snn_snia_vs_nonia'],
@@ -112,7 +112,7 @@ def early_sn_candidates(
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_early_sn_candidates.filter.early_sn_candidates'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_kn_candidates/filter.py

@@ -65,7 +65,7 @@ def kn_candidates_(
 
     Examples
     ----------
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = kn_candidates_(
     ...     pdf['rf_kn_vs_nonkn'],
     ...     pdf['rf_snia_vs_nonia'],
@@ -143,7 +143,7 @@ def kn_candidates(
     ----------
     >>> from fink_utils.spark.utils import concat_col
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
 
     >>> to_expand = [
     ...    'jd', 'fid', 'magpsf', 'sigmapsf',

fink_filters/filter_known_tde/filter.py

@@ -49,7 +49,7 @@ def known_tde_(objectId, ra, dec, radius_arcsec=pd.Series([5])) -> pd.Series:
 
     Examples
     ----------
-    >>> pdf = pd.read_parquet('datatest_tde')
+    >>> pdf = pd.read_parquet('datatest/tde')
     >>> classification = known_tde_(
     ...     pdf['objectId'],
     ...     pdf['candidate'].apply(lambda x: x['ra']),
@@ -126,7 +126,7 @@ def known_tde(objectId, ra, dec) -> pd.Series:
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest_tde')
+    >>> df = spark.read.format('parquet').load('datatest/tde')
     >>> f = 'fink_filters.filter_known_tde.filter.known_tde'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_microlensing_candidates/filter.py

@@ -37,7 +37,7 @@ def microlensing_candidates_(mulens) -> pd.Series:
 
     Examples
     ----------
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = microlensing_candidates_(pdf['mulens'])
     >>> print(pdf[classification]['objectId'].values)
     []
@@ -67,7 +67,7 @@ def microlensing_candidates(mulens) -> pd.Series:
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_microlensing_candidates.filter.microlensing_candidates'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_mm_module/filter.py

@@ -269,5 +269,5 @@ def f_gw_bronze_events(fink_class, observatory, rb):
 
     # Run the test suite
     globs = globals()
-    globs["grb_output_data"] = "datatest_grb/grb_test_data.parquet"
+    globs["grb_output_data"] = "datatest/grb/grb_test_data.parquet"
     spark_unit_tests(globs)

fink_filters/filter_orphan_grb_candidates/filter.py

@@ -84,7 +84,7 @@ def orphan_grb(jd, jdstarthist, cjdc, cfidc, cssnamenrc, cmagpsfc):
     ----------
     >>> from fink_utils.spark.utils import concat_col
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
 
     >>> to_expand = ['jd', 'fid', 'ssnamenr', 'magpsf']
 

fink_filters/filter_rate_based_kn_candidates/filter.py

@@ -73,6 +73,7 @@ def perform_classification(
     cjdc, cfidc, cmagpsfc, csigmapsfc, cmagnrc, csigmagnrc, cmagzpscic: Spark DataFrame Columns
         Columns containing history of fid, magpsf, sigmapsf, magnr, sigmagnr,
         magzpsci, isdiffpos as arrays
+
     Returns
     ----------
     out: pandas.Series of bool
@@ -215,6 +216,7 @@ def rate_based_kn_candidates_(
     cjdc, cfidc, cmagpsfc, csigmapsfc, cmagnrc, csigmagnrc, cmagzpscic: Spark DataFrame Columns
         Columns containing history of fid, magpsf, sigmapsf, magnr, sigmagnr,
         magzpsci, isdiffpos as arrays
+
     Returns
     ----------
     out: pandas.Series of bool
@@ -270,6 +272,7 @@ def rate_based_kn_candidates(
     cjdc, cfidc, cmagpsfc, csigmapsfc, cmagnrc, csigmagnrc, cmagzpscic: Spark DataFrame Columns
         Columns containing history of fid, magpsf, sigmapsf, magnr, sigmagnr,
         magzpsci, isdiffpos as arrays
+
     Returns
     ----------
     out: pandas.Series of bool
@@ -280,7 +283,7 @@ def rate_based_kn_candidates(
     ----------
     >>> from fink_utils.spark.utils import concat_col
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
 
     >>> to_expand = ['jd', 'fid', 'magpsf', 'sigmapsf', 'magnr', 'sigmagnr', 'magzpsci', 'isdiffpos']
 

fink_filters/filter_rrlyr/filter.py

@@ -39,7 +39,7 @@ def rrlyr(cdsxmatch: Any) -> pd.Series:
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_rrlyr.filter.rrlyr'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_simbad_candidates/filter.py

@@ -35,7 +35,7 @@ def simbad_candidates_(cdsxmatch) -> pd.Series:
 
     Examples
     ----------
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = simbad_candidates_(pdf['cdsxmatch'])
     >>> nalerts = len(pdf[classification]['objectId'])
     >>> print(nalerts)
@@ -80,7 +80,7 @@ def simbad_candidates(cdsxmatch) -> pd.Series:
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_simbad_candidates.filter.simbad_candidates'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_simbad_grav_candidates/filter.py

@@ -35,7 +35,7 @@ def simbad_grav_candidates_(cdsxmatch) -> pd.Series:
 
     Examples
     ----------
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = simbad_grav_candidates_(pdf['cdsxmatch'])
     >>> nalerts = len(pdf[classification]['objectId'])
     >>> print(nalerts)
@@ -80,7 +80,7 @@ def simbad_grav_candidates(cdsxmatch) -> pd.Series:
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_simbad_grav_candidates.filter.simbad_grav_candidates'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_sn_candidates/filter.py

@@ -56,7 +56,7 @@ def sn_candidates_(
 
     Examples
     ----------
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = sn_candidates_(
     ...     pdf['cdsxmatch'],
     ...     pdf['snn_snia_vs_nonia'],
@@ -95,7 +95,7 @@ def sn_candidates(
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_sn_candidates.filter.sn_candidates'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_sso_fink_candidates/filter.py

@@ -35,7 +35,7 @@ def sso_fink_candidates_(roid) -> pd.Series:
 
     Examples
     ----------
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = sso_fink_candidates_(pdf['roid'])
     >>> print(len(pdf[classification]['objectId'].values))
     3
@@ -64,7 +64,7 @@ def sso_fink_candidates(roid) -> pd.Series:
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_sso_fink_candidates.filter.sso_fink_candidates'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_sso_ztf_candidates/filter.py

@@ -35,7 +35,7 @@ def sso_ztf_candidates_(roid) -> pd.Series:
 
     Examples
     ----------
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = sso_ztf_candidates_(pdf['roid'])
     >>> print(len(pdf[classification]['objectId'].values))
     3
@@ -64,7 +64,7 @@ def sso_ztf_candidates(roid) -> pd.Series:
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_sso_ztf_candidates.filter.sso_ztf_candidates'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_tracklet_candidates/filter.py

@@ -35,7 +35,7 @@ def tracklet_candidates_(tracklet) -> pd.Series:
 
     Examples
     ----------
-    >>> pdf = pd.read_parquet('datatest')
+    >>> pdf = pd.read_parquet('datatest/regular')
     >>> classification = tracklet_candidates_(pdf['tracklet'])
     >>> print(len(pdf[classification]['objectId'].values))
     2
@@ -64,7 +64,7 @@ def tracklet_candidates(tracklet) -> pd.Series:
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_tracklet_candidates.filter.tracklet_candidates'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())

fink_filters/filter_yso_candidates/filter.py

@@ -40,7 +40,7 @@ def yso_candidates(cdsxmatch: Any) -> pd.Series:
     Examples
     ----------
     >>> from fink_utils.spark.utils import apply_user_defined_filter
-    >>> df = spark.read.format('parquet').load('datatest')
+    >>> df = spark.read.format('parquet').load('datatest/regular')
     >>> f = 'fink_filters.filter_yso_candidates.filter.yso_candidates'
     >>> df = apply_user_defined_filter(df, f)
     >>> print(df.count())