From fb8926f1ded990d091e658e6bac06d14f81c1fdc Mon Sep 17 00:00:00 2001 From: Julien Date: Mon, 11 Dec 2023 11:40:32 +0100 Subject: [PATCH] Update code dependencies (#764) * Update dependencies * Remove unused dependency file * Update syntax for AL Ia for elasticc (requires fink-science>=5.50) * Test Sentinel on a single image * Refactor * Dependencies for Fink-MM --- .github/workflows/test.yml | 2 +- deps/requirements-science-no-deps.txt | 13 --------- deps/requirements-science.txt | 33 ++++++++++++++++++++--- deps/requirements-test.txt | 3 +++ deps/requirements.txt | 38 +++++++++++++++------------ fink_broker/science.py | 20 +++----------- 6 files changed, 59 insertions(+), 50 deletions(-) delete mode 100644 deps/requirements-science-no-deps.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 378b3510..9a70dd25 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: - container: ["julienpeloton/fink-ci:prod", "julienpeloton/fink-ci:dev"] + container: ["julienpeloton/fink-ci:dev"] container: image: ${{ matrix.container }} diff --git a/deps/requirements-science-no-deps.txt b/deps/requirements-science-no-deps.txt deleted file mode 100644 index 6e3d287b..00000000 --- a/deps/requirements-science-no-deps.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Fink-fat --e git+https://github.com/FusRoman/fink-fat.git@1e840a47372cc3fe31df3cf7b41caf60144aff63#egg=fink-fat --install-option="--no-deps" - -# Fink_GRB --e git+https://github.com/FusRoman/fink-mm.git@f3dea27b6aeb3bc0529458958f43e436dfd5efa0#egg=fink-mm --install-option="--no-deps" - -# Installation of torch without GPU support (lighter) --f https://download.pytorch.org/whl/torch_stable.html -torch==1.12.0+cpu - -# Installation of astronet --e git+https://github.com/tallamjr/astronet.git#egg=astronet --install-option="--no-deps" -tensorflow==2.9.2 diff --git a/deps/requirements-science.txt b/deps/requirements-science.txt index 712dae6b..bf263f36 100644 --- a/deps/requirements-science.txt +++ b/deps/requirements-science.txt @@ -1,6 +1,7 @@ # Active learning --e git+https://github.com/emilleishida/fink_sn_activelearning.git@bf8d4e263e02d42781642f872f7bc030c24792bc#egg=actsnfink +-e git+https://github.com/emilleishida/fink_sn_activelearning.git@4f46b3a1e29de45793125452974e71e92c1ea454#egg=actsnfink -e git+https://github.com/COINtoolbox/ActSNClass.git@2c61da91a9d13834d39804fc35aeb3245ba20755#egg=actsnclass +joblib==1.2.0 # microlensing -e git+https://github.com/JulienPeloton/LIA.git@4513654b15a0760312eff2661b0fcf4989171ce3#egg=LIA @@ -13,10 +14,10 @@ colorama seaborn # SNAD -light-curve>=0.7.3 +light-curve[full] # AGN -iminuit==2.11.2 +iminuit==2.21.0 # kilonova @ sklearn==1.0.2 -e git+https://github.com/b-biswas/kndetect@kndetect#egg=kndetect @@ -27,3 +28,29 @@ torch==1.12.0+cpu # CATS dependencies tensorflow==2.9.2 +tensorflow_addons==0.18.0 + +# Anomalies +onnx==1.12.0 +onnxruntime + +# fink-fat +fink_fat @ https://github.com/FusRoman/fink-fat/archive/v0.16.0.zip --install-option="--no-deps" +sbpy + +# Fink MM +fink_MM @ https://github.com/FusRoman/Fink_MM/archive/v0.16.10.zip --install-option="--no-deps" +gcn-kafka==0.3.0 +voevent-parse==1.0.3 +tabulate==0.9.0 +jsonschema==4.6.0 +astropy_healpix==0.7 +hdfs==2.7.2 +importlib-resources==5.9.0 +pandera==0.14.5 + +# Installation of astronet +-e git+https://github.com/tallamjr/astronet.git#egg=astronet --install-option="--no-deps" +imbalanced-learn==0.7.0 +optuna==2.3.0 +george diff --git a/deps/requirements-test.txt b/deps/requirements-test.txt index d76706ea..e7757760 100644 --- a/deps/requirements-test.txt +++ b/deps/requirements-test.txt @@ -1,2 +1,5 @@ mypy pytest + +coverage>=4.2 +coveralls \ No newline at end of file diff --git a/deps/requirements.txt b/deps/requirements.txt index d279406a..dbf60e22 100644 --- a/deps/requirements.txt +++ b/deps/requirements.txt @@ -1,26 +1,30 @@ # Dependencies for Fink -numpy>=1.17 -coverage>=4.2 -coveralls +numpy==1.23.5 pandas==1.3.5 -# > 1.9. see https://github.com/edenhill/librdkafka/issues/3263 -confluent-kafka==1.9.2 -avro-python3 -Cython -fastavro==1.6.0 pyarrow==9.0.0 -slackclient -astropy -astroquery -fink_filters>=3.21 -git+https://github.com/astrolabsoftware/fink-science@5.0.0 -fink-utils>=0.13.8 scipy scikit-learn==1.0.2 scikit-image +Cython + +# Astro healpy +astropy +astroquery + +# > 1.9. see https://github.com/edenhill/librdkafka/issues/3263 +confluent-kafka==1.9.2 +avro-python3 +fastavro==1.6.0 + +# Fink core +fink_filters>=3.22 +git+https://github.com/astrolabsoftware/fink-science@5.5.0 +fink-utils>=0.13.8 +fink-spins>=0.3.3 fink-tns>=0.9 -protobuf==3.20 -tensorflow_addons -slack_sdk + +# Misc +slack_sdk>=3.21.3 aiohttp +py4j diff --git a/fink_broker/science.py b/fink_broker/science.py index 07d52a49..78c1052c 100644 --- a/fink_broker/science.py +++ b/fink_broker/science.py @@ -51,7 +51,7 @@ from fink_science.ad_features.processor import extract_features_ad from fink_science.anomaly_detection.processor import anomaly_score - from fink_science.random_forest_snia.processor import rfscore_sigmoid_elasticc + from fink_science.random_forest_snia.processor import rfscore_rainbow_elasticc from fink_science.snn.processor import snn_ia_elasticc, snn_broad_elasticc from fink_science.cats.processor import predict_nn from fink_science.agn.processor import agn_elasticc @@ -450,15 +450,11 @@ def apply_science_modules_elasticc(df: DataFrame) -> DataFrame: _LOG.info("New processor: EarlySN") args = ['cmidPointTai', 'cfilterName', 'cpsFlux', 'cpsFluxErr'] - - # fake cdsxmatch and nobs - args += [F.col('diaObject.ra'), F.col('diaObject.decl')] - args += [F.col('diaObject.hostgal_ra'), F.col('diaObject.hostgal_dec')] + args += [F.col('diaSource.snr')] args += [F.col('diaObject.hostgal_snsep')] args += [F.col('diaObject.hostgal_zphot')] - args += [F.col('diaObject.hostgal_zphot_err')] - df = df.withColumn('rf_snia_vs_nonia', rfscore_sigmoid_elasticc(*args)) + df = df.withColumn('rf_snia_vs_nonia', rfscore_rainbow_elasticc(*args)) # Apply level one processor: superNNova _LOG.info("New processor: supernnova - Ia") @@ -509,7 +505,7 @@ def apply_science_modules_elasticc(df: DataFrame) -> DataFrame: df = df.withColumn('cats_broad_class', mapping_cats_general_expr[df['argmax']]) df = df.withColumn('cats_broad_max_prob', F.array_max(df['cbpf_preds'])) - # AGN + # AGN & SLSN args_forced = [ 'diaObject.diaObjectId', 'cmidPointTai', 'cpsFlux', 'cpsFluxErr', 'cfilterName', 'diaSource.ra', 'diaSource.decl', @@ -517,14 +513,6 @@ def apply_science_modules_elasticc(df: DataFrame) -> DataFrame: 'diaObject.hostgal_ra', 'diaObject.hostgal_dec' ] df = df.withColumn('rf_agn_vs_nonagn', agn_elasticc(*args_forced)) - - # SLSN - args_forced = [ - 'diaObject.diaObjectId', 'cmidPointTai', 'cpsFlux', 'cpsFluxErr', 'cfilterName', - 'diaSource.ra', 'diaSource.decl', - 'diaObject.hostgal_zphot', 'diaObject.hostgal_zphot_err', - 'diaObject.hostgal_ra', 'diaObject.hostgal_dec' - ] df = df.withColumn('rf_slsn_vs_nonslsn', slsn_elasticc(*args_forced)) # Drop temp columns