SSHG1G2 (#908)

* Add new command in the scheduler to compute the SSOFT using the SSHG1G2 model * disable cache for rocks
astrolabsoftware · Oct 24, 2024 · 5b44af7 · 5b44af7
1 parent 3b326ae
commit 5b44af7
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 4 deletions.
diff --git a/deps/requirements-science-no-deps.txt b/deps/requirements-science-no-deps.txt
@@ -3,7 +3,7 @@ fink_fat @ https://github.com/FusRoman/fink-fat/archive/v0.16.1.zip
 sbpy
 
 # Fink MM
-fink_MM @ https://github.com/FusRoman/Fink_MM/archive/v0.20.1.zip
+fink_MM @ https://github.com/FusRoman/Fink_MM/archive/v0.21.0.zip
 
 # Installation of torch without GPU support (lighter)
 -f https://download.pytorch.org/whl/torch_stable.html

diff --git a/deps/requirements-science.txt b/deps/requirements-science.txt
@@ -1,7 +1,7 @@
 # Fink-science dependencies
 # WARNING: fink-science pip module is deprecated and fink-science is now installed from source
 
-git+https://github.com/astrolabsoftware/fink-science@5.20.0
+git+https://github.com/astrolabsoftware/fink-science@5.21.0
 
 # xmatch_cds
 line_profiler==4.1.3

diff --git a/deps/requirements.txt b/deps/requirements.txt
@@ -19,7 +19,7 @@ fastavro==1.6.0
 
 # Fink core
 fink_filters>=3.37
-fink-utils>=0.28.0
+fink-utils>=0.29.3
 fink-spins>=0.3.8
 fink-tns>=0.9
 

diff --git a/scheduler/launch_ssoft.sh b/scheduler/launch_ssoft.sh
@@ -1,8 +1,25 @@
 #!/bin/bash
+# Copyright 2019-2024 AstroLab Software
+# Author: Julien Peloton
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 set -e
 
 source ~/.bash_profile
 
+# re-download latest information
+export ROCKS_CACHE_DIR="no-cache"
+
 NCORES=100
 AGGREGATE="--pre_aggregate_data"
 EXTRA_OPT=
@@ -27,7 +44,21 @@ spark-submit \
     --conf spark.kryoserializer.buffer.max=512m\
     ${PYTHON_EXTRA_FILE}\
     ${FINK_HOME}/bin/generate_ssoft.py \
-    -model SHG1G2 $AGGREGATE $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_SHG1G2.log 2>&1
+    -model SSHG1G2 $AGGREGATE $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_SSHG1G2.log 2>&1
+
+spark-submit \
+    --master mesos://vm-75063.lal.in2p3.fr:5050 \
+    --conf spark.mesos.principal=$MESOS_PRINCIPAL \
+    --conf spark.mesos.secret=$MESOS_SECRET \
+    --conf spark.mesos.role=$MESOS_ROLE \
+    --conf spark.executorEnv.HOME='/home/julien.peloton'\
+    --driver-memory 8G --executor-memory 4G \
+    --conf spark.cores.max=$NCORES --conf spark.executor.cores=2 \
+    --conf spark.sql.execution.arrow.pyspark.enabled=true\
+    --conf spark.kryoserializer.buffer.max=512m\
+    ${PYTHON_EXTRA_FILE}\
+    ${FINK_HOME}/bin/generate_ssoft.py \
+    -model SHG1G2 $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_SHG1G2.log 2>&1
 
 spark-submit \
     --master mesos://vm-75063.lal.in2p3.fr:5050 \
@@ -61,6 +92,7 @@ sudo su livy <<'EOF'
 source ~/.bashrc
 YEAR=`date +"%Y"`
 MONTH=`date +"%m"`
+/opt/hadoop-2/bin/hdfs dfs -put ssoft_SSHG1G2_${YEAR}.${MONTH}.parquet SSOFT/
 /opt/hadoop-2/bin/hdfs dfs -put ssoft_SHG1G2_${YEAR}.${MONTH}.parquet SSOFT/
 /opt/hadoop-2/bin/hdfs dfs -put ssoft_HG1G2_${YEAR}.${MONTH}.parquet SSOFT/
 /opt/hadoop-2/bin/hdfs dfs -put ssoft_HG_${YEAR}.${MONTH}.parquet SSOFT/