diff --git a/deps/requirements-science-no-deps.txt b/deps/requirements-science-no-deps.txt index dfa55f2f..f700c05d 100644 --- a/deps/requirements-science-no-deps.txt +++ b/deps/requirements-science-no-deps.txt @@ -3,7 +3,7 @@ fink_fat @ https://github.com/FusRoman/fink-fat/archive/v0.16.1.zip sbpy # Fink MM -fink_MM @ https://github.com/FusRoman/Fink_MM/archive/v0.20.1.zip +fink_MM @ https://github.com/FusRoman/Fink_MM/archive/v0.21.0.zip # Installation of torch without GPU support (lighter) -f https://download.pytorch.org/whl/torch_stable.html diff --git a/deps/requirements-science.txt b/deps/requirements-science.txt index e66a28e6..6688c9cd 100644 --- a/deps/requirements-science.txt +++ b/deps/requirements-science.txt @@ -1,7 +1,7 @@ # Fink-science dependencies # WARNING: fink-science pip module is deprecated and fink-science is now installed from source -git+https://github.com/astrolabsoftware/fink-science@5.20.0 +git+https://github.com/astrolabsoftware/fink-science@5.21.0 # xmatch_cds line_profiler==4.1.3 diff --git a/deps/requirements.txt b/deps/requirements.txt index 0da71837..27a83954 100644 --- a/deps/requirements.txt +++ b/deps/requirements.txt @@ -19,7 +19,7 @@ fastavro==1.6.0 # Fink core fink_filters>=3.37 -fink-utils>=0.28.0 +fink-utils>=0.29.3 fink-spins>=0.3.8 fink-tns>=0.9 diff --git a/scheduler/launch_ssoft.sh b/scheduler/launch_ssoft.sh index 8a8ebbc3..69a86ddf 100755 --- a/scheduler/launch_ssoft.sh +++ b/scheduler/launch_ssoft.sh @@ -1,8 +1,25 @@ #!/bin/bash +# Copyright 2019-2024 AstroLab Software +# Author: Julien Peloton +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. set -e source ~/.bash_profile +# re-download latest information +export ROCKS_CACHE_DIR="no-cache" + NCORES=100 AGGREGATE="--pre_aggregate_data" EXTRA_OPT= @@ -27,7 +44,21 @@ spark-submit \ --conf spark.kryoserializer.buffer.max=512m\ ${PYTHON_EXTRA_FILE}\ ${FINK_HOME}/bin/generate_ssoft.py \ - -model SHG1G2 $AGGREGATE $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_SHG1G2.log 2>&1 + -model SSHG1G2 $AGGREGATE $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_SSHG1G2.log 2>&1 + +spark-submit \ + --master mesos://vm-75063.lal.in2p3.fr:5050 \ + --conf spark.mesos.principal=$MESOS_PRINCIPAL \ + --conf spark.mesos.secret=$MESOS_SECRET \ + --conf spark.mesos.role=$MESOS_ROLE \ + --conf spark.executorEnv.HOME='/home/julien.peloton'\ + --driver-memory 8G --executor-memory 4G \ + --conf spark.cores.max=$NCORES --conf spark.executor.cores=2 \ + --conf spark.sql.execution.arrow.pyspark.enabled=true\ + --conf spark.kryoserializer.buffer.max=512m\ + ${PYTHON_EXTRA_FILE}\ + ${FINK_HOME}/bin/generate_ssoft.py \ + -model SHG1G2 $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_SHG1G2.log 2>&1 spark-submit \ --master mesos://vm-75063.lal.in2p3.fr:5050 \ @@ -61,6 +92,7 @@ sudo su livy <<'EOF' source ~/.bashrc YEAR=`date +"%Y"` MONTH=`date +"%m"` +/opt/hadoop-2/bin/hdfs dfs -put ssoft_SSHG1G2_${YEAR}.${MONTH}.parquet SSOFT/ /opt/hadoop-2/bin/hdfs dfs -put ssoft_SHG1G2_${YEAR}.${MONTH}.parquet SSOFT/ /opt/hadoop-2/bin/hdfs dfs -put ssoft_HG1G2_${YEAR}.${MONTH}.parquet SSOFT/ /opt/hadoop-2/bin/hdfs dfs -put ssoft_HG_${YEAR}.${MONTH}.parquet SSOFT/