Update code dependencies (#764)

* Update dependencies

* Remove unused dependency file

* Update syntax for AL Ia for elasticc (requires fink-science>=5.50)

* Test Sentinel on a single image

* Refactor

* Dependencies for Fink-MM

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        container: ["julienpeloton/fink-ci:prod", "julienpeloton/fink-ci:dev"]
+        container: ["julienpeloton/fink-ci:dev"]
 
     container:
       image: ${{ matrix.container }}

deps/requirements-science.txt

@@ -1,6 +1,7 @@
 # Active learning
--e git+https://github.com/emilleishida/fink_sn_activelearning.git@bf8d4e263e02d42781642f872f7bc030c24792bc#egg=actsnfink
+-e git+https://github.com/emilleishida/fink_sn_activelearning.git@4f46b3a1e29de45793125452974e71e92c1ea454#egg=actsnfink
 -e git+https://github.com/COINtoolbox/ActSNClass.git@2c61da91a9d13834d39804fc35aeb3245ba20755#egg=actsnclass
+joblib==1.2.0
 
 # microlensing
 -e git+https://github.com/JulienPeloton/LIA.git@4513654b15a0760312eff2661b0fcf4989171ce3#egg=LIA
@@ -13,10 +14,10 @@ colorama
 seaborn
 
 # SNAD
-light-curve>=0.7.3
+light-curve[full]
 
 # AGN
-iminuit==2.11.2
+iminuit==2.21.0
 
 # kilonova @ sklearn==1.0.2
 -e git+https://github.com/b-biswas/kndetect@kndetect#egg=kndetect
@@ -27,3 +28,29 @@ torch==1.12.0+cpu
 
 # CATS dependencies
 tensorflow==2.9.2
+tensorflow_addons==0.18.0
+
+# Anomalies
+onnx==1.12.0
+onnxruntime
+
+# fink-fat
+fink_fat @ https://github.com/FusRoman/fink-fat/archive/v0.16.0.zip --install-option="--no-deps"
+sbpy
+
+# Fink MM
+fink_MM @ https://github.com/FusRoman/Fink_MM/archive/v0.16.10.zip --install-option="--no-deps"
+gcn-kafka==0.3.0
+voevent-parse==1.0.3
+tabulate==0.9.0
+jsonschema==4.6.0
+astropy_healpix==0.7
+hdfs==2.7.2
+importlib-resources==5.9.0
+pandera==0.14.5
+
+# Installation of astronet
+-e git+https://github.com/tallamjr/astronet.git#egg=astronet --install-option="--no-deps"
+imbalanced-learn==0.7.0
+optuna==2.3.0
+george

deps/requirements-test.txt

@@ -1,2 +1,5 @@
 mypy
 pytest
+
+coverage>=4.2
+coveralls

deps/requirements.txt

@@ -1,26 +1,30 @@
 # Dependencies for Fink
-numpy>=1.17
-coverage>=4.2
-coveralls
+numpy==1.23.5
 pandas==1.3.5
-# > 1.9. see https://github.com/edenhill/librdkafka/issues/3263
-confluent-kafka==1.9.2
-avro-python3
-Cython
-fastavro==1.6.0
 pyarrow==9.0.0
-slackclient
-astropy
-astroquery
-fink_filters>=3.21
-git+https://github.com/astrolabsoftware/[email protected]
-fink-utils>=0.13.8
 scipy
 scikit-learn==1.0.2
 scikit-image
+Cython
+
+# Astro
 healpy
+astropy
+astroquery
+
+# > 1.9. see https://github.com/edenhill/librdkafka/issues/3263
+confluent-kafka==1.9.2
+avro-python3
+fastavro==1.6.0
+
+# Fink core
+fink_filters>=3.22
+git+https://github.com/astrolabsoftware/[email protected]
+fink-utils>=0.13.8
+fink-spins>=0.3.3
 fink-tns>=0.9
-protobuf==3.20
-tensorflow_addons
-slack_sdk
+
+# Misc
+slack_sdk>=3.21.3
 aiohttp
+py4j

fink_broker/science.py

@@ -51,7 +51,7 @@
     from fink_science.ad_features.processor import extract_features_ad
     from fink_science.anomaly_detection.processor import anomaly_score
 
-    from fink_science.random_forest_snia.processor import rfscore_sigmoid_elasticc
+    from fink_science.random_forest_snia.processor import rfscore_rainbow_elasticc
     from fink_science.snn.processor import snn_ia_elasticc, snn_broad_elasticc
     from fink_science.cats.processor import predict_nn
     from fink_science.agn.processor import agn_elasticc
@@ -450,15 +450,11 @@ def apply_science_modules_elasticc(df: DataFrame) -> DataFrame:
 
     _LOG.info("New processor: EarlySN")
     args = ['cmidPointTai', 'cfilterName', 'cpsFlux', 'cpsFluxErr']
-
-    # fake cdsxmatch and nobs
-    args += [F.col('diaObject.ra'), F.col('diaObject.decl')]
-    args += [F.col('diaObject.hostgal_ra'), F.col('diaObject.hostgal_dec')]
+    args += [F.col('diaSource.snr')]
     args += [F.col('diaObject.hostgal_snsep')]
     args += [F.col('diaObject.hostgal_zphot')]
-    args += [F.col('diaObject.hostgal_zphot_err')]
 
-    df = df.withColumn('rf_snia_vs_nonia', rfscore_sigmoid_elasticc(*args))
+    df = df.withColumn('rf_snia_vs_nonia', rfscore_rainbow_elasticc(*args))
 
     # Apply level one processor: superNNova
     _LOG.info("New processor: supernnova - Ia")
@@ -509,22 +505,14 @@ def apply_science_modules_elasticc(df: DataFrame) -> DataFrame:
     df = df.withColumn('cats_broad_class', mapping_cats_general_expr[df['argmax']])
     df = df.withColumn('cats_broad_max_prob', F.array_max(df['cbpf_preds']))
 
-    # AGN
+    # AGN & SLSN
     args_forced = [
         'diaObject.diaObjectId', 'cmidPointTai', 'cpsFlux', 'cpsFluxErr', 'cfilterName',
         'diaSource.ra', 'diaSource.decl',
         'diaObject.hostgal_zphot', 'diaObject.hostgal_zphot_err',
         'diaObject.hostgal_ra', 'diaObject.hostgal_dec'
     ]
     df = df.withColumn('rf_agn_vs_nonagn', agn_elasticc(*args_forced))
-
-    # SLSN
-    args_forced = [
-        'diaObject.diaObjectId', 'cmidPointTai', 'cpsFlux', 'cpsFluxErr', 'cfilterName',
-        'diaSource.ra', 'diaSource.decl',
-        'diaObject.hostgal_zphot', 'diaObject.hostgal_zphot_err',
-        'diaObject.hostgal_ra', 'diaObject.hostgal_dec'
-    ]
     df = df.withColumn('rf_slsn_vs_nonslsn', slsn_elasticc(*args_forced))
 
     # Drop temp columns