diff --git a/scheduler/README.md b/scheduler/README.md index 7bd3ed66..4191076f 100644 --- a/scheduler/README.md +++ b/scheduler/README.md @@ -3,21 +3,23 @@ Operations for the night N start at 21.45pm UTC on the night N-1. There are 2 cronjobs scheduled: ```bash -# Paris time +# Paris time @ VD # Fink real-time 45 23 * * * /home/julien.peloton/fink-broker/scheduler/launch_fink.sh # Database service -05 20 * * * /home/julien.peloton/fink-broker/scheduler/database_service.sh +35 20 * * * /home/julien.peloton/fink-broker/scheduler/database_service.sh +30 21 * * * /home/julien.peloton/fink-broker/scheduler/database_auxilliary.sh # Fink MM 0 01 * * * /home/julien.peloton/Fink_MM/scheduler/science2grb.sh 0 01 * * * /home/julien.peloton/Fink_MM/scheduler/grb2distribution.sh 1 17 * * * /home/julien.peloton/Fink_MM/scheduler/science2grb_offline.sh -# SSOFT +# SSOFT - once a month 0 0 1 * * /home/julien.peloton/fink-broker/scheduler/launch_ssoft.sh +0 12 1 * * /home/julien.peloton/fink-broker/scheduler/launch_sso_resolver.sh ``` The first script is for live operations: diff --git a/scheduler/launch_sso_resolver.sh b/scheduler/launch_sso_resolver.sh new file mode 100644 index 00000000..09d6c1de --- /dev/null +++ b/scheduler/launch_sso_resolver.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +source ~/.bash_profile + +source ${FINK_HOME}/conf_cluster/fink.conf.ztf_stream2raw + +NCORES=8 + +FINK_VERSION=`fink --version` +PYTHON_VERSION=`python -c "import platform; print(platform.python_version()[:3])"` +PYTHON_EXTRA_FILE="--py-files ${FINK_HOME}/dist/fink_broker-${FINK_VERSION}- py${PYTHON_VERSION}.egg" + +spark-submit \ + --master mesos://vm-75063.lal.in2p3.fr:5050 \ + --conf spark.mesos.principal=$MESOS_PRINCIPAL \ + --conf spark.mesos.secret=$MESOS_SECRET \ + --conf spark.mesos.role=$MESOS_ROLE \ + --conf spark.executorEnv.HOME='/home/julien.peloton'\ + --driver-memory 8G --executor-memory 4G \ + --conf spark.cores.max=$NCORES --conf spark.executor.cores=2 \ + --conf spark.sql.execution.arrow.pyspark.enabled=true\ + --conf spark.kryoserializer.buffer.max=512m\ + --packages ${FINK_PACKAGES} --jars ${FINK_JARS} ${PYTHON_EXTRA_FILE} \ + ${FINK_HOME}/bin/index_sso_resolver.py +~ diff --git a/scheduler/launch_ssoft.sh b/scheduler/launch_ssoft.sh new file mode 100755 index 00000000..8a8ebbc3 --- /dev/null +++ b/scheduler/launch_ssoft.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -e + +source ~/.bash_profile + +NCORES=100 +AGGREGATE="--pre_aggregate_data" +EXTRA_OPT= + +# In case of trouble +#AGGREGATE="" +#EXTRA_OPT="-version 2023.12" + +FINK_VERSION=`fink --version` +PYTHON_VERSION=`python -c "import platform; print(platform.python_version()[:3])"` +PYTHON_EXTRA_FILE="--py-files ${FINK_HOME}/dist/fink_broker-${FINK_VERSION}-py${PYTHON_VERSION}.egg" + +spark-submit \ + --master mesos://vm-75063.lal.in2p3.fr:5050 \ + --conf spark.mesos.principal=$MESOS_PRINCIPAL \ + --conf spark.mesos.secret=$MESOS_SECRET \ + --conf spark.mesos.role=$MESOS_ROLE \ + --conf spark.executorEnv.HOME='/home/julien.peloton'\ + --driver-memory 8G --executor-memory 4G \ + --conf spark.cores.max=$NCORES --conf spark.executor.cores=2 \ + --conf spark.sql.execution.arrow.pyspark.enabled=true\ + --conf spark.kryoserializer.buffer.max=512m\ + ${PYTHON_EXTRA_FILE}\ + ${FINK_HOME}/bin/generate_ssoft.py \ + -model SHG1G2 $AGGREGATE $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_SHG1G2.log 2>&1 + +spark-submit \ + --master mesos://vm-75063.lal.in2p3.fr:5050 \ + --conf spark.mesos.principal=$MESOS_PRINCIPAL \ + --conf spark.mesos.secret=$MESOS_SECRET \ + --conf spark.mesos.role=$MESOS_ROLE \ + --conf spark.executorEnv.HOME='/home/julien.peloton'\ + --driver-memory 8G --executor-memory 4G \ + --conf spark.cores.max=$NCORES --conf spark.executor.cores=2 \ + --conf spark.sql.execution.arrow.pyspark.enabled=true\ + --conf spark.kryoserializer.buffer.max=512m\ + ${PYTHON_EXTRA_FILE}\ + ${FINK_HOME}/bin/generate_ssoft.py \ + -model HG1G2 $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_HG1G2.log 2>&1 + +spark-submit \ + --master mesos://vm-75063.lal.in2p3.fr:5050 \ + --conf spark.mesos.principal=$MESOS_PRINCIPAL \ + --conf spark.mesos.secret=$MESOS_SECRET \ + --conf spark.mesos.role=$MESOS_ROLE \ + --conf spark.executorEnv.HOME='/home/julien.peloton'\ + --driver-memory 8G --executor-memory 4G \ + --conf spark.cores.max=$NCORES --conf spark.executor.cores=2 \ + --conf spark.sql.execution.arrow.pyspark.enabled=true\ + --conf spark.kryoserializer.buffer.max=512m\ + ${PYTHON_EXTRA_FILE}\ + ${FINK_HOME}/bin/generate_ssoft.py \ + -model HG $EXTRA_OPT > ${FINK_HOME}/broker_logs/ssoft_HG.log 2>&1 + +sudo su livy <<'EOF' +source ~/.bashrc +YEAR=`date +"%Y"` +MONTH=`date +"%m"` +/opt/hadoop-2/bin/hdfs dfs -put ssoft_SHG1G2_${YEAR}.${MONTH}.parquet SSOFT/ +/opt/hadoop-2/bin/hdfs dfs -put ssoft_HG1G2_${YEAR}.${MONTH}.parquet SSOFT/ +/opt/hadoop-2/bin/hdfs dfs -put ssoft_HG_${YEAR}.${MONTH}.parquet SSOFT/ +EOF + +mv ssoft_*.parquet /spark_mongo_tmp/julien.peloton/ssoft/