Skip to content

Commit

Permalink
fixed unit tests for Slurm script generation syntax for all solvers
Browse files Browse the repository at this point in the history
  • Loading branch information
@jcschaff
jcschaff committed Sep 26, 2024
1 parent 2806650 commit 61d21e7
Show file tree
Hide file tree
Showing 12 changed files with 405 additions and 1,167 deletions.
14 changes: 10 additions & 4 deletions vcell-server/src/test/java/cbit/vcell/message/server/htc/slurm/SlurmProxyTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,9 @@ public void setup()
setProperty(PropertyLoader.mongodbPortExternal, "30019");
setProperty(PropertyLoader.mongodbDatabase, "test");
setProperty(PropertyLoader.vcellSoftwareVersion, "Rel_Version_7.6.0_build_28");
setProperty(PropertyLoader.vcellbatch_singularity_image, "/state/partition1/singularityImages/ghcr.io_virtualcell_vcell-batch_d6825f4.img");
setProperty(PropertyLoader.slurm_tmpdir, "/scratch/vcell");
setProperty(PropertyLoader.slurm_central_singularity_dir, "/share/apps/vcell3/singularityImages");
setProperty(PropertyLoader.slurm_local_singularity_dir, "/state/partition1/singularityImages");
setProperty(PropertyLoader.slurm_singularity_cachedir, "/share/apps/vcell3/singularity/cachdir");
setProperty(PropertyLoader.slurm_singularity_pullfolder, "/share/apps/vcell3/singularity/pullfolder");
setProperty(PropertyLoader.slurm_singularity_module_name, "singularity/vcell-3.10.0");
setProperty(PropertyLoader.simDataDirArchiveExternal, "/share/apps/vcell12/users");
setProperty(PropertyLoader.simDataDirArchiveInternal, "/share/apps/vcell12/users");
Expand All @@ -72,7 +71,14 @@ public void setup()
setProperty(PropertyLoader.simulationPreprocessor, "JavaPreprocessor64");

setProperty(PropertyLoader.primarySimDataDirInternalProperty, "/share/apps/vcell3/users");
setProperty(PropertyLoader.vcellopt_singularity_image, "/state/partition1/singularityImages/ghcr.io_virtualcell_vcell-opt_d6825f4.img");
setProperty(PropertyLoader.htc_vcellopt_docker_name, "ghcr.io/virtualcell/vcell-opt:7.6.0.43");

setProperty(PropertyLoader.htc_vcellfvsolver_solver_list, "Smoldyn,SundialsPDE");
setProperty(PropertyLoader.htc_vcellfvsolver_docker_name, "ghcr.io/virtualcell/vcell-fvsolver:v0.9.4");
setProperty(PropertyLoader.htc_vcellsolvers_solver_list, "HybridMilstein,StochGibson,Smoldyn,MovingBoundary,SundialsPDE,CombinedSundials,NFSim");
setProperty(PropertyLoader.htc_vcellsolvers_docker_name, "ghcr.io/virtualcell/vcell-solvers:v0.8.1.2");
setProperty(PropertyLoader.htc_vcellbatch_solver_list, "RungeKuttaFehlberg,HybridMilstein,StochGibson,Langevin,AdamsMoulton,Smoldyn,MovingBoundary,SundialsPDE,CombinedSundials,NFSim");
setProperty(PropertyLoader.htc_vcellbatch_docker_name, "ghcr.io/virtualcell/vcell-batch:7.6.0.43");
}

@AfterEach
Expand Down
129 changes: 36 additions & 93 deletions vcell-server/src/test/resources/slurm_fixtures/adams_moulton/V_REL_274633859_0_0.slurm.sub
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,130 +8,73 @@
#SBATCH --mem=4096M
#SBATCH --no-kill
#SBATCH --no-requeue
# VCell SlurmProxy memory limit source=Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB
# VCell SlurmProxy memory limit source='Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB'


#BEGIN---------SlurmProxy.generateScript():slurmInitSingularity----------
set -x

TMPDIR=/scratch/vcell
echo "using TMPDIR=$TMPDIR"
if [ ! -e $TMPDIR ]; then mkdir -p $TMPDIR ; fi
echo `hostname`

export MODULEPATH=/isg/shared/modulefiles:/tgcapps/modulefiles

source /usr/share/Modules/init/bash

module load singularity/vcell-3.10.0

export SINGULARITY_CACHEDIR=/share/apps/vcell3/singularity/cachdir
export SINGULARITY_PULLFOLDER=/share/apps/vcell3/singularity/pullfolder
echo "job running on host `hostname -f`"

echo "id is `id`"

echo "bash version is `bash --version`"
date

echo ENVIRONMENT
env

container_prefix=
if command -v singularity >/dev/null 2>&1; then
#
# Copy of singularity image will be downloaded if not found in /state/partition1/singularityImages/ghcr.io_virtualcell_vcell-batch_d6825f4.img
#
localSingularityImage=/state/partition1/singularityImages/ghcr.io_virtualcell_vcell-batch_d6825f4.img
if [ ! -e "$localSingularityImage" ]; then
echo "local singularity image $localSingularityImage not found, trying to download to hpc from "/share/apps/vcell3/singularityImages/ghcr.io_virtualcell_vcell-batch_d6825f4.img
mkdir -p /state/partition1/singularityImages
singularitytempfile=$(mktemp -up /share/apps/vcell3/singularityImages)
flock -E 100 -n /tmp/vcellSingularityLock_Rel_Version_7.6.0_build_28.lock sh -c "cp /share/apps/vcell3/singularityImages/ghcr.io_virtualcell_vcell-batch_d6825f4.img ${singularitytempfile} ; mv -n ${singularitytempfile} /state/partition1/singularityImages/ghcr.io_virtualcell_vcell-batch_d6825f4.img"
theStatus=$?
if [ $theStatus -eq 100 ]
then
echo "lock in use, waiting for lock owner to copy singularityImage"
let c=0
until [ -f $localSingularityImage ]
do
sleep 3
let c=c+1
if [ $c -eq 20 ]
then
echo "Exceeded wait time for lock owner to copy singularityImage"
break
fi
done
else
if [ $theStatus -eq 0 ]
then
echo copy succeeded
else
echo copy failed
fi
fi
rm -f ${singularitytempfile}
if [ ! -e "$localSingularityImage" ]; then
echo "Failed to copy $localSingularityImage to hpc from central"
exit 1
else
echo successful copy from /share/apps/vcell3/singularityImages/ghcr.io_virtualcell_vcell-batch_d6825f4.img to /state/partition1/singularityImages/ghcr.io_virtualcell_vcell-batch_d6825f4.img
fi
fi
container_prefix="singularity run --containall --bind /share/apps/vcell3/users:/simdata --bind /share/apps/vcell7/users:/simdata_secondary --bind /share/apps/vcell12/users:/share/apps/vcell12/users --bind /share/apps/vcell3/htclogs:/htclogs --bind /scratch/vcell:/solvertmp $localSingularityImage --env java_mem_Xmx=4096M --env jmshost_sim_internal=rke-wn-01.cam.uchc.edu --env jmsport_sim_internal=31618 --env jmsrestport_sim_internal=30163 --env jmsuser=clientUser --env jmspswd=dummy --env jmsblob_minsize=100000 --env mongodbhost_internal=rke-wn-01.cam.uchc.edu --env mongodbport_internal=30019 --env mongodb_database=test --env primary_datadir_external=/share/apps/vcell3/users --env secondary_datadir_external=/share/apps/vcell7/users --env htclogdir_external=/share/apps/vcell3/htclogs --env softwareVersion=Rel_Version_7.6.0_build_28 --env serverid=REL "
else
echo "Required singularity command not found (maybe 'module load singularity/vcell-3.10.0' command didn't work) "
exit 1
fi
echo "container_prefix is '${container_prefix}'"
echo "3 date=`date`"
#END---------SlurmProxy.generateScript():slurmInitSingularity----------
container_bindings="--bind /share/apps/vcell3/users:/simdata "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /share/apps/vcell7/users:/simdata_secondary "
container_env="--env java_mem_Xmx=4096M "
container_env+="--env jmshost_sim_internal=rke-wn-01.cam.uchc.edu "
container_env+="--env jmsport_sim_internal=31618 "
container_env+="--env jmsrestport_sim_internal=30163 "
container_env+="--env jmsuser=clientUser "
container_env+="--env jmspswd=dummy "
container_env+="--env jmsblob_minsize=100000 "
container_env+="--env mongodbhost_internal=rke-wn-01.cam.uchc.edu "
container_env+="--env mongodbport_internal=30019 "
container_env+="--env mongodb_database=test "
container_env+="--env primary_datadir_external=/share/apps/vcell3/users "
container_env+="--env secondary_datadir_external=/share/apps/vcell7/users "
container_env+="--env htclogdir_external=/share/apps/vcell3/htclogs "
container_env+="--env softwareVersion=Rel_Version_7.6.0_build_28 "
container_env+="--env serverid=REL "
solver_docker_name=ghcr.io/virtualcell/vcell-batch:7.6.0.43
solver_container_prefix="singularity run --containall ${container_bindings} ${container_env} docker://${solver_docker_name}"
batch_docker_name=ghcr.io/virtualcell/vcell-batch:7.6.0.43
batch_container_prefix="singularity run --containall ${container_bindings} ${container_env} docker://${batch_docker_name}"


#BEGIN---------SlurmProxy.generateScript():sendFailureMsg----------
sendFailureMsg() {
echo ${container_prefix} --msg-userid clientUser --msg-password dummy --msg-host rke-wn-01.cam.uchc.edu --msg-port 31618 --msg-job-host `hostname` --msg-job-userid schaff --msg-job-simkey 274633859 --msg-job-jobindex 0 --msg-job-taskid 0 --msg-job-errmsg "$1" SendErrorMsg
${container_prefix} --msg-userid clientUser --msg-password dummy --msg-host rke-wn-01.cam.uchc.edu --msg-port 31618 --msg-job-host `hostname` --msg-job-userid schaff --msg-job-simkey 274633859 --msg-job-jobindex 0 --msg-job-taskid 0 --msg-job-errmsg "$1" SendErrorMsg
${batch_container_prefix} --msg-userid clientUser --msg-password dummy --msg-host rke-wn-01.cam.uchc.edu --msg-port 31618 --msg-job-host `hostname` --msg-job-userid schaff --msg-job-simkey 274633859 --msg-job-jobindex 0 --msg-job-taskid 0 --msg-job-errmsg "$1" SendErrorMsg
stat=$?
if [[ $stat -ne 0 ]]; then
echo 'failed to send error message, retcode=$stat'
else
echo 'sent failure message'
fi
}
#END---------SlurmProxy.generateScript():sendFailureMsg----------
#BEGIN---------SlurmProxy.generateScript():hasExitProcessor----------


callExitProcessor( ) {
echo exitCommand = ${container_prefix}JavaPostprocessor64 274633859 schaff 17 0 0 $1 /share/apps/vcell3/htclogs/V_REL_274633859_0_0.slurm.sub
${container_prefix}JavaPostprocessor64 274633859 schaff 17 0 0 $1 /share/apps/vcell3/htclogs/V_REL_274633859_0_0.slurm.sub
"${batch_container_prefix} JavaPostprocessor64 274633859 schaff 17 0 0 $1 /share/apps/vcell3/htclogs/V_REL_274633859_0_0.slurm.sub "
}
#END---------SlurmProxy.generateScript():hasExitProcessor----------
echo
echo "1 date=`date`"

echo
#BEGIN---------SlurmProxy.generateScript():ExecutableCommand----------JavaSimExe64
echo "testing existance of native exe '/share/apps/vcell3/nativesolvers/JavaSimExe64' which overrides container invocations"
nativeExe=/share/apps/vcell3/nativesolvers/JavaSimExe64
if [ -e "${nativeExe}" ]; then
cmd_prefix="/share/apps/vcell3/nativesolvers/"
else
cmd_prefix="$container_prefix"
fi
echo "cmd_prefix is '${cmd_prefix}'"
echo "5 date=`date`"
echo command = ${cmd_prefix}JavaSimExe64 /share/apps/vcell3/users/schaff/SimID_274633859_0__0.simtask.xml /share/apps/vcell3/users/schaff
command="${cmd_prefix}JavaSimExe64 /share/apps/vcell3/users/schaff/SimID_274633859_0__0.simtask.xml /share/apps/vcell3/users/schaff "
$command

"${solver_container_prefix} JavaSimExe64 /share/apps/vcell3/users/schaff/SimID_274633859_0__0.simtask.xml /share/apps/vcell3/users/schaff "
stat=$?
echo ${cmd_prefix}JavaSimExe64 /share/apps/vcell3/users/schaff/SimID_274633859_0__0.simtask.xml /share/apps/vcell3/users/schaff returned $stat
echo returned $stat
if [ $stat -ne 0 ]; then
callExitProcessor $stat
echo returning $stat to Slurm
exit $stat
fi
#END---------SlurmProxy.generateScript():ExecutableCommand----------JavaSimExe64
callExitProcessor 0


#Following commands (if any) are read by JavaPostProcessor64
callExitProcessor 0


#Following commands (if any) are read by JavaPostProcessor64
Loading

0 comments on commit 61d21e7

Please sign in to comment.