cleanup

.gitignore

@@ -12,6 +12,3 @@ build/
 # Distribution / packaging
 dist/
 babs/VERSION
-
-# e2e testdata
-.testdata*

Makefile

@@ -7,13 +7,9 @@ setup-user:
 e2e: clean
 	./tests/e2e-slurm/main.sh
 
-# TODO testdata variable
 clean:
-	podman stop slurm 2>/dev/null || true
-	podman rm slurm 2>/dev/null || true
-	[ -e .testdata/babs_test_project/toybidsapp-container ] && \
+	@ podman stop slurm 2>/dev/null || true
+	@ podman rm slurm 2>/dev/null || true
+	@[ -e .testdata/babs_test_project/toybidsapp-container ] && \
 		datalad remove -d .testdata/babs_test_project/toybidsapp-container --reckless kill || :
 	rm -rf .testdata
-
-logs:
-	cat .testdata/ci-logs/*

babs/babs.py

@@ -2778,14 +2778,10 @@ def generate_job_submit_template(self, yaml_path, babs, system, test=False):
             env_flags = "-v DSLOCKFILE=" + babs.analysis_path + "/.SGE_datalad_lock"
         elif system.type == "slurm":
             submit_head = "sbatch"
-            # TODO: asmacdo
             env_flags = "--export=DSLOCKFILE=" + babs.analysis_path + "/.SLURM_datalad_lock"
         else:
             warnings.warn("not supporting systems other than sge...")
 
-        # TODO: rm asmacdo hack
-        # env_flags = env_flags + f",MINICONDA_PATH={os.getenv('MINICONDA_PATH')}"
-
         # Check if the bash file already exist:
         if op.exists(yaml_path):
             os.remove(yaml_path)  # remove it

babs/utils.py

@@ -1674,7 +1674,6 @@ def submit_one_test_job(analysis_path, type_system, flag_print_message=True):
                               stdout=subprocess.PIPE)
 
     proc_cmd.check_returncode()
-    print(f"Return code: {proc_cmd.returncode}")
     msg = proc_cmd.stdout.decode('utf-8')
 
     if type_system == "sge":

tests/e2e-slurm/container/babs-user-script.sh

@@ -4,6 +4,7 @@ SUBPROJECT_NAME=test_project
 
 set -eu
 
+echo "=============================================================="
 echo "We are now running as user $(whoami)"
 echo "DEBUG: MINICONDA_PATH=${MINICONDA_PATH}"
 echo "DEBUG: TESTDATA=${TESTDATA}"
@@ -113,43 +114,4 @@ else
 fi
 
 babs-merge --project_root "${PWD}"/test_project/
-
-
-# TODO: we need to fail if there is a failed job
-# fi
-
-# sleep 10
-# babs-status --project_root "${PWD}"/test_project/
-# sleep 10
-# babs-status --project_root "${PWD}"/test_project/
-# sleep 10
-# babs-status --project_root "${PWD}"/test_project/
-# sleep 10
-# babs-status --project_root "${PWD}"/test_project/
-#
-# babs-submit --project_root "${PWD}"/test_project/
-#
-# babs-status --project_root "${PWD}"/test_project/
-# sleep 30s
-# babs-status --project_root "${PWD}"/test_project/
-#
-# echo "Print job logs--------------------------------------------"
-# find "${PWD}"/test_project/analysis/logs/* -type f -print -exec cat {} \;
-# echo "end job logs--------------------------------------------"
-# # TODO: babs-check-status-job
-#
-# # TODO babs-merge
-#
-# popd
-# # /tests/e2e-slurm/babs-tests.sh
-# # podman exec  \
-# # 	-e MINICONDA_PATH=${MINICONDA_PATH} \
-# # 	slurm \
-# # 	${PWD}/tests/e2e-slurm/babs-tests.sh
-# #
-#
-#
-# echo "--------------------------"
-# echo "     HUZZZZZZAHHHHHH!!!!!!"
-# echo "--------------------------"
-#
+echo "PASSED: e2e walkthrough successfull!"

tests/e2e-slurm/container/ensure-env.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 #
-# exported for use in inner-slurm.sh
 if [ -z "${MINICONDA_PATH:-}" ]; then
     if hash conda; then
         # We don't need the return value, we already catch the error

tests/e2e-slurm/container/walkthrough-tests.sh

@@ -1,21 +1,13 @@
 #!/bin/bash -i
-
+# Here we perform all actions that must be done as root inside the container and then
+# execute the walkthrough as BABS_USER
 set -eu
 
-# add that outside user
-# groupadd --gid "$GID" "$USER"  && useradd --uid $UID --gid "$GID" "$USER"
-
-# Install singularity inside the container
-yum update -y && yum install -y epel-release &&  yum update -y &&  yum install -y singularity-runtime apptainer
-#
-# git version
-# git config user.name > /dev/null || git config --system user.name "e2e slurm"
-# git config user.email > /dev/null || git config --system user.email "[email protected]"
-# git config --system --add safe.directory '*'
-
 export TESTDATA=/opt/testdata
 BABS_USER=testuser
 
+# Install singularity inside the container
+yum update -y && yum install -y epel-release &&  yum update -y &&  yum install -y singularity-runtime apptainer
 
 # Wait for slurm to be up
 max_retries=10

tests/e2e-slurm/install-babs.sh

@@ -10,6 +10,4 @@ conda install -c conda-forge datalad git git-annex -y
 pip install datalad_container
 pip install datalad-osf
 
-# TODO non-dynamic for prod
-# pip install .
-pip install -e .
+pip install .

tests/e2e-slurm/main.sh

@@ -21,6 +21,7 @@ TAG=23.11.07 # TODO
 FQDN_IMAGE=${REGISTRY}/${HUBUSER}/${REPO}:${TAG}
 THIS_DIR="$(readlink -f "$0" | xargs dirname )"
 
+# Sets MINICONDA_PATH
 . tests/e2e-slurm/container/ensure-env.sh
 
 if [ "$MINICONDA_PATH/envs/$CONDA_DEFAULT_ENV/bin/babs-init" != "$(which babs-init)" ]; then
@@ -29,12 +30,9 @@ if [ "$MINICONDA_PATH/envs/$CONDA_DEFAULT_ENV/bin/babs-init" != "$(which babs-in
     exit 1
 fi
 
-stop_container () {
-	podman stop slurm || true
-}
-
 echo "Success, we are in the conda env with babs-init!"
- # Because babs is dev-installed from here. TODO: we can remove if we remove -e from pip install
+
+# PWD shared so babs can be optionally be installed with develop install
 podman run -it --rm \
 	--name slurm \
 	--hostname slurmctl  \
@@ -45,6 +43,3 @@ podman run -it --rm \
 	-v "${THIS_DIR}/container:/opt/outer:ro,Z" \
 	"${FQDN_IMAGE}" \
 	/bin/bash -c ". /opt/outer/walkthrough-tests.sh"
-
-	#/bin/bash -c ". /opt/outer/walkthrough-tests.sh && bash" # TODO remove, for debug only
-# trap stop_container EXIT