From 29bca2dfe13ac11d17abb8f5e9bb8a55c4e45577 Mon Sep 17 00:00:00 2001 From: JulienPeloton Date: Thu, 17 Oct 2024 08:34:32 +0200 Subject: [PATCH 1/2] Add SSO index table script --- scheduler/launch_sso_resolver.sh | 26 ++++++++++++ scheduler/launch_ssoft.sh | 69 ++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 scheduler/launch_sso_resolver.sh create mode 100755 scheduler/launch_ssoft.sh diff --git a/scheduler/launch_sso_resolver.sh b/scheduler/launch_sso_resolver.sh new file mode 100644 index 00000000..09d6c1de --- /dev/null +++ b/scheduler/launch_sso_resolver.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +source ~/.bash_profile + +source ${FINK_HOME}/conf_cluster/fink.conf.ztf_stream2raw + +NCORES=8 + +FINK_VERSION=`fink --version` +PYTHON_VERSION=`python -c "import platform; print(platform.python_version()[:3])"` +PYTHON_EXTRA_FILE="--py-files ${FINK_HOME}/dist/fink_broker-${FINK_VERSION}- py${PYTHON_VERSION}.egg" + +spark-submit \ + --master mesos://vm-75063.lal.in2p3.fr:5050 \ + --conf spark.mesos.principal=$MESOS_PRINCIPAL \ + --conf spark.mesos.secret=$MESOS_SECRET \ + --conf spark.mesos.role=$MESOS_ROLE \ + --conf spark.executorEnv.HOME='/home/julien.peloton'\ + --driver-memory 8G --executor-memory 4G \ + --conf spark.cores.max=$NCORES --conf spark.executor.cores=2 \ + --conf spark.sql.execution.arrow.pyspark.enabled=true\ + --conf spark.kryoserializer.buffer.max=512m\ + --packages ${FINK_PACKAGES} --jars ${FINK_JARS} ${PYTHON_EXTRA_FILE} \ + ${FINK_HOME}/bin/index_sso_resolver.py +~ diff --git a/scheduler/launch_ssoft.sh b/scheduler/launch_ssoft.sh new file mode 100755 index 00000000..8a8ebbc3 --- /dev/null +++ b/scheduler/launch_ssoft.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -e + +source ~/.bash_profile + +NCORES=100 +AGGREGATE="--pre_aggregate_data" +EXTRA_OPT= + +# In case of trouble +#AGGREGATE="" +#EXTRA_OPT="-version 2023.12" + +FINK_VERSION=`fink --version` +PYTHON_VERSION=`python -c "import platform; print(platform.python_version()[:3])"` +PYTHON_EXTRA_FILE="--py-files ${FINK_HOME}/dist/fink_broker-${FINK_VERSION}-py${PYTHON_VERSION}.egg" + +spark-submit \ + --master mesos://vm-75063.lal.in2p3.fr:5050 \ + --conf spark.mesos.principal=$MESOS_PRINCIPAL \ + --conf spark.mesos.secret=$MESOS_SECRET \ + --conf spark.mesos.role=$MESOS_ROLE \ + --conf spark.executorEnv.HOME='/home/julien.peloton'\ + --driver-memory 8G --executor-memory 4G \ + --conf spark.cores.max=$NCORES --conf spark.executor.cores=2 \ + --conf spark.sql.execution.arrow.pyspark.enabled=true\ + --conf spark.kryoserializer.buffer.max=512m\ + ${PYTHON_EXTRA_FILE}\ + ${FINK_HOME}/bin/generate_ssoft.py \ + -model SHG1G2 $AGGREGATE $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_SHG1G2.log 2>&1 + +spark-submit \ + --master mesos://vm-75063.lal.in2p3.fr:5050 \ + --conf spark.mesos.principal=$MESOS_PRINCIPAL \ + --conf spark.mesos.secret=$MESOS_SECRET \ + --conf spark.mesos.role=$MESOS_ROLE \ + --conf spark.executorEnv.HOME='/home/julien.peloton'\ + --driver-memory 8G --executor-memory 4G \ + --conf spark.cores.max=$NCORES --conf spark.executor.cores=2 \ + --conf spark.sql.execution.arrow.pyspark.enabled=true\ + --conf spark.kryoserializer.buffer.max=512m\ + ${PYTHON_EXTRA_FILE}\ + ${FINK_HOME}/bin/generate_ssoft.py \ + -model HG1G2 $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_HG1G2.log 2>&1 + +spark-submit \ + --master mesos://vm-75063.lal.in2p3.fr:5050 \ + --conf spark.mesos.principal=$MESOS_PRINCIPAL \ + --conf spark.mesos.secret=$MESOS_SECRET \ + --conf spark.mesos.role=$MESOS_ROLE \ + --conf spark.executorEnv.HOME='/home/julien.peloton'\ + --driver-memory 8G --executor-memory 4G \ + --conf spark.cores.max=$NCORES --conf spark.executor.cores=2 \ + --conf spark.sql.execution.arrow.pyspark.enabled=true\ + --conf spark.kryoserializer.buffer.max=512m\ + ${PYTHON_EXTRA_FILE}\ + ${FINK_HOME}/bin/generate_ssoft.py \ + -model HG $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_HG.log 2>&1 + +sudo su livy <<'EOF' +source ~/.bashrc +YEAR=`date +"%Y"` +MONTH=`date +"%m"` +/opt/hadoop-2/bin/hdfs dfs -put ssoft_SHG1G2_${YEAR}.${MONTH}.parquet SSOFT/ +/opt/hadoop-2/bin/hdfs dfs -put ssoft_HG1G2_${YEAR}.${MONTH}.parquet SSOFT/ +/opt/hadoop-2/bin/hdfs dfs -put ssoft_HG_${YEAR}.${MONTH}.parquet SSOFT/ +EOF + +mv ssoft_*.parquet /spark_mongo_tmp/julien.peloton/ssoft/ From e04cd5752ad7f8e628f5bb7a56d29063fb00d89c Mon Sep 17 00:00:00 2001 From: JulienPeloton Date: Thu, 17 Oct 2024 08:39:35 +0200 Subject: [PATCH 2/2] Update the cron file --- scheduler/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scheduler/README.md b/scheduler/README.md index 7bd3ed66..4191076f 100644 --- a/scheduler/README.md +++ b/scheduler/README.md @@ -3,21 +3,23 @@ Operations for the night N start at 21.45pm UTC on the night N-1. There are 2 cronjobs scheduled: ```bash -# Paris time +# Paris time @ VD # Fink real-time 45 23 * * * /home/julien.peloton/fink-broker/scheduler/launch_fink.sh # Database service -05 20 * * * /home/julien.peloton/fink-broker/scheduler/database_service.sh +35 20 * * * /home/julien.peloton/fink-broker/scheduler/database_service.sh +30 21 * * * /home/julien.peloton/fink-broker/scheduler/database_auxilliary.sh # Fink MM 0 01 * * * /home/julien.peloton/Fink_MM/scheduler/science2grb.sh 0 01 * * * /home/julien.peloton/Fink_MM/scheduler/grb2distribution.sh 1 17 * * * /home/julien.peloton/Fink_MM/scheduler/science2grb_offline.sh -# SSOFT +# SSOFT - once a month 0 0 1 * * /home/julien.peloton/fink-broker/scheduler/launch_ssoft.sh +0 12 1 * * /home/julien.peloton/fink-broker/scheduler/launch_sso_resolver.sh ``` The first script is for live operations: