diff --git a/.gitignore b/.gitignore index cf103c72..de79a6da 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,3 @@ build/ # Distribution / packaging dist/ babs/VERSION - -# e2e testdata -.testdata* diff --git a/Makefile b/Makefile index 03949a2e..a6143630 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,9 @@ setup-user: e2e: clean ./tests/e2e-slurm/main.sh -# TODO testdata variable clean: - podman stop slurm 2>/dev/null || true - podman rm slurm 2>/dev/null || true - [ -e .testdata/babs_test_project/toybidsapp-container ] && \ + @ podman stop slurm 2>/dev/null || true + @ podman rm slurm 2>/dev/null || true + @[ -e .testdata/babs_test_project/toybidsapp-container ] && \ datalad remove -d .testdata/babs_test_project/toybidsapp-container --reckless kill || : rm -rf .testdata - -logs: - cat .testdata/ci-logs/* diff --git a/babs/babs.py b/babs/babs.py index 4c9025a2..f9719fe1 100644 --- a/babs/babs.py +++ b/babs/babs.py @@ -2778,14 +2778,10 @@ def generate_job_submit_template(self, yaml_path, babs, system, test=False): env_flags = "-v DSLOCKFILE=" + babs.analysis_path + "/.SGE_datalad_lock" elif system.type == "slurm": submit_head = "sbatch" - # TODO: asmacdo env_flags = "--export=DSLOCKFILE=" + babs.analysis_path + "/.SLURM_datalad_lock" else: warnings.warn("not supporting systems other than sge...") - # TODO: rm asmacdo hack - # env_flags = env_flags + f",MINICONDA_PATH={os.getenv('MINICONDA_PATH')}" - # Check if the bash file already exist: if op.exists(yaml_path): os.remove(yaml_path) # remove it diff --git a/babs/utils.py b/babs/utils.py index f6ca0c50..d8103475 100644 --- a/babs/utils.py +++ b/babs/utils.py @@ -1674,7 +1674,6 @@ def submit_one_test_job(analysis_path, type_system, flag_print_message=True): stdout=subprocess.PIPE) proc_cmd.check_returncode() - print(f"Return code: {proc_cmd.returncode}") msg = proc_cmd.stdout.decode('utf-8') if type_system == "sge": diff --git a/tests/e2e-slurm/container/babs-user-script.sh b/tests/e2e-slurm/container/babs-user-script.sh index 166a178b..6848ac63 100755 --- a/tests/e2e-slurm/container/babs-user-script.sh +++ b/tests/e2e-slurm/container/babs-user-script.sh @@ -4,6 +4,7 @@ SUBPROJECT_NAME=test_project set -eu +echo "==============================================================" echo "We are now running as user $(whoami)" echo "DEBUG: MINICONDA_PATH=${MINICONDA_PATH}" echo "DEBUG: TESTDATA=${TESTDATA}" @@ -113,43 +114,4 @@ else fi babs-merge --project_root "${PWD}"/test_project/ - - -# TODO: we need to fail if there is a failed job -# fi - -# sleep 10 -# babs-status --project_root "${PWD}"/test_project/ -# sleep 10 -# babs-status --project_root "${PWD}"/test_project/ -# sleep 10 -# babs-status --project_root "${PWD}"/test_project/ -# sleep 10 -# babs-status --project_root "${PWD}"/test_project/ -# -# babs-submit --project_root "${PWD}"/test_project/ -# -# babs-status --project_root "${PWD}"/test_project/ -# sleep 30s -# babs-status --project_root "${PWD}"/test_project/ -# -# echo "Print job logs--------------------------------------------" -# find "${PWD}"/test_project/analysis/logs/* -type f -print -exec cat {} \; -# echo "end job logs--------------------------------------------" -# # TODO: babs-check-status-job -# -# # TODO babs-merge -# -# popd -# # /tests/e2e-slurm/babs-tests.sh -# # podman exec \ -# # -e MINICONDA_PATH=${MINICONDA_PATH} \ -# # slurm \ -# # ${PWD}/tests/e2e-slurm/babs-tests.sh -# # -# -# -# echo "--------------------------" -# echo " HUZZZZZZAHHHHHH!!!!!!" -# echo "--------------------------" -# +echo "PASSED: e2e walkthrough successfull!" diff --git a/tests/e2e-slurm/container/ensure-env.sh b/tests/e2e-slurm/container/ensure-env.sh index fee3162b..352ed3a7 100755 --- a/tests/e2e-slurm/container/ensure-env.sh +++ b/tests/e2e-slurm/container/ensure-env.sh @@ -1,6 +1,5 @@ #!/bin/bash # -# exported for use in inner-slurm.sh if [ -z "${MINICONDA_PATH:-}" ]; then if hash conda; then # We don't need the return value, we already catch the error diff --git a/tests/e2e-slurm/container/walkthrough-tests.sh b/tests/e2e-slurm/container/walkthrough-tests.sh index 4c36f436..1c57f2d9 100755 --- a/tests/e2e-slurm/container/walkthrough-tests.sh +++ b/tests/e2e-slurm/container/walkthrough-tests.sh @@ -1,21 +1,13 @@ #!/bin/bash -i - +# Here we perform all actions that must be done as root inside the container and then +# execute the walkthrough as BABS_USER set -eu -# add that outside user -# groupadd --gid "$GID" "$USER" && useradd --uid $UID --gid "$GID" "$USER" - -# Install singularity inside the container -yum update -y && yum install -y epel-release && yum update -y && yum install -y singularity-runtime apptainer -# -# git version -# git config user.name > /dev/null || git config --system user.name "e2e slurm" -# git config user.email > /dev/null || git config --system user.email "fake@example.com" -# git config --system --add safe.directory '*' - export TESTDATA=/opt/testdata BABS_USER=testuser +# Install singularity inside the container +yum update -y && yum install -y epel-release && yum update -y && yum install -y singularity-runtime apptainer # Wait for slurm to be up max_retries=10 diff --git a/tests/e2e-slurm/install-babs.sh b/tests/e2e-slurm/install-babs.sh index b9531fc8..a0674743 100755 --- a/tests/e2e-slurm/install-babs.sh +++ b/tests/e2e-slurm/install-babs.sh @@ -10,6 +10,4 @@ conda install -c conda-forge datalad git git-annex -y pip install datalad_container pip install datalad-osf -# TODO non-dynamic for prod -# pip install . -pip install -e . +pip install . diff --git a/tests/e2e-slurm/main.sh b/tests/e2e-slurm/main.sh index c50b002b..f4e35a4b 100755 --- a/tests/e2e-slurm/main.sh +++ b/tests/e2e-slurm/main.sh @@ -21,6 +21,7 @@ TAG=23.11.07 # TODO FQDN_IMAGE=${REGISTRY}/${HUBUSER}/${REPO}:${TAG} THIS_DIR="$(readlink -f "$0" | xargs dirname )" +# Sets MINICONDA_PATH . tests/e2e-slurm/container/ensure-env.sh if [ "$MINICONDA_PATH/envs/$CONDA_DEFAULT_ENV/bin/babs-init" != "$(which babs-init)" ]; then @@ -29,12 +30,9 @@ if [ "$MINICONDA_PATH/envs/$CONDA_DEFAULT_ENV/bin/babs-init" != "$(which babs-in exit 1 fi -stop_container () { - podman stop slurm || true -} - echo "Success, we are in the conda env with babs-init!" - # Because babs is dev-installed from here. TODO: we can remove if we remove -e from pip install + +# PWD shared so babs can be optionally be installed with develop install podman run -it --rm \ --name slurm \ --hostname slurmctl \ @@ -45,6 +43,3 @@ podman run -it --rm \ -v "${THIS_DIR}/container:/opt/outer:ro,Z" \ "${FQDN_IMAGE}" \ /bin/bash -c ". /opt/outer/walkthrough-tests.sh" - - #/bin/bash -c ". /opt/outer/walkthrough-tests.sh && bash" # TODO remove, for debug only -# trap stop_container EXIT