From 98dbf33147ed029800dd43a73ee5c64e83feda7b Mon Sep 17 00:00:00 2001 From: Lon Blauvelt Date: Wed, 16 Dec 2020 14:14:33 -0800 Subject: [PATCH] Refactor logging and bioio. (#3351) * Refactor logging. * Refactor addOptions. * Refactor. * Refactor. * Refactor. * Add lower. * Add lower. * Account parse stats arg correctly. * Add back Cactus-dependent naming. * Cruft. * Legacy log levels. * Updates. * Delete options.py * Updates. * Copyright. * Copyright. * Don't use __all__. * Update setup. * Replace deprecated imp library. * Run make sort_imports. * Run make format. * Switch back to imp. * Setup update. * Setup update. * Make sure sorted imports use VERTICAL. * Clean up launch cluster. * Refactor launch cluster. * Run black after sorting imports. * Refactor spot worker bid. * Fix test that is suddenly now running. * Better message. * SLURM Error? * Import error. * Fix import. * Resolve error and add deprecation warnings. * Add skipped doc tests. * Update. * Imports. * Suppress extraneous logging while testing. * Cruft. * Set logging. Remove docker in docker test. * Check upgrade to latest dev dependencies. * Update pytest version. * Update pytest version. * Update. * Suppress logging with env var. * Dev reqs versioning. * Add integration test. * Cruft. * Fix path. * Clean up utils test. --- .gitlab-ci.yml | 36 ++- Makefile | 12 +- docker/Dockerfile.py | 2 +- requirements-dev.txt | 16 + setup.py | 66 ++-- setup_gitlab_docker.py | 13 + setup_gitlab_ssh.py | 13 + src/toil/__init__.py | 8 +- src/toil/batchSystems/__init__.py | 6 +- src/toil/batchSystems/abstractBatchSystem.py | 2 +- .../abstractGridEngineBatchSystem.py | 2 +- src/toil/batchSystems/gridengine.py | 6 +- src/toil/batchSystems/htcondor.py | 4 +- src/toil/batchSystems/kubernetes.py | 20 +- src/toil/batchSystems/lsf.py | 29 +- src/toil/batchSystems/mesos/__init__.py | 2 +- src/toil/batchSystems/mesos/batchSystem.py | 11 +- src/toil/batchSystems/mesos/conftest.py | 2 +- src/toil/batchSystems/mesos/executor.py | 14 +- src/toil/batchSystems/options.py | 142 +++++---- src/toil/batchSystems/parasol.py | 7 +- src/toil/batchSystems/registry.py | 2 +- src/toil/batchSystems/singleMachine.py | 12 +- src/toil/batchSystems/slurm.py | 8 +- src/toil/batchSystems/torque.py | 7 +- src/toil/common.py | 172 +++++----- src/toil/cwl/__init__.py | 1 - src/toil/cwl/conftest.py | 2 +- src/toil/cwl/cwltoil.py | 6 +- src/toil/deferred.py | 9 +- src/toil/fileStores/__init__.py | 11 +- src/toil/fileStores/abstractFileStore.py | 5 +- src/toil/fileStores/cachingFileStore.py | 7 +- src/toil/fileStores/nonCachingFileStore.py | 15 +- src/toil/job.py | 30 +- src/toil/jobStores/abstractJobStore.py | 10 +- src/toil/jobStores/aws/jobStore.py | 168 +++++----- src/toil/jobStores/aws/utils.py | 27 +- src/toil/jobStores/conftest.py | 2 +- src/toil/jobStores/fileJobStore.py | 6 +- src/toil/jobStores/googleJobStore.py | 2 +- src/toil/jobStores/utils.py | 3 - src/toil/leader.py | 28 +- src/toil/lib/bioio.py | 296 ++---------------- src/toil/lib/compatibility.py | 8 - src/toil/lib/context.py | 19 +- src/toil/lib/docker.py | 19 +- src/toil/lib/ec2.py | 66 ++-- src/toil/lib/ec2nodes.py | 2 +- src/toil/lib/encryption/__init__.py | 2 +- src/toil/lib/encryption/_dummy.py | 2 +- src/toil/lib/encryption/_nacl.py | 2 +- src/toil/lib/exceptions.py | 1 - src/toil/lib/expando.py | 3 +- src/toil/lib/generatedEC2Lists.py | 2 +- src/toil/lib/iterables.py | 2 +- src/toil/lib/memoize.py | 2 +- src/toil/lib/objects.py | 2 +- src/toil/lib/resources.py | 33 ++ src/toil/lib/retry.py | 8 +- src/toil/lib/threading.py | 41 ++- src/toil/lib/throttle.py | 2 +- src/toil/provisioners/.gitignore | 2 - src/toil/provisioners/__init__.py | 69 +++- src/toil/provisioners/abstractProvisioner.py | 2 +- src/toil/provisioners/aws/__init__.py | 112 ++----- src/toil/provisioners/aws/awsProvisioner.py | 31 +- src/toil/provisioners/clusterScaler.py | 5 +- src/toil/provisioners/gceProvisioner.py | 2 +- src/toil/provisioners/node.py | 2 +- src/toil/realtimeLogger.py | 28 +- src/toil/resource.py | 28 +- src/toil/serviceManager.py | 2 +- src/toil/statsAndLogging.py | 155 ++++++--- src/toil/test/__init__.py | 61 ++-- src/toil/test/batchSystems/__init__.py | 2 +- src/toil/test/batchSystems/batchSystemTest.py | 41 ++- .../batchSystems/parasolTestSupport.py | 16 +- src/toil/test/cwl/conftest.py | 2 +- src/toil/test/cwl/cwlTest.py | 16 +- .../test/docs/scripts/example_alwaysfail.py | 2 - .../docs/scripts/example_cachingbenchmark.py | 3 - src/toil/test/docs/scriptsTest.py | 64 ++-- src/toil/test/jobStores/__init__.py | 2 +- src/toil/test/jobStores/jobStoreTest.py | 11 +- src/toil/test/lib/dockerTest.py | 10 +- .../test/mesos/MesosDataStructuresTest.py | 2 +- src/toil/test/mesos/__init__.py | 2 +- src/toil/test/mesos/helloWorld.py | 2 +- src/toil/test/mesos/stress.py | 2 +- src/toil/test/provisioners/__init__.py | 2 +- src/toil/test/provisioners/aws/__init__.py | 2 +- .../provisioners/aws/awsProvisionerTest.py | 21 +- .../test/provisioners/clusterScalerTest.py | 8 +- .../test/provisioners/gceProvisionerTest.py | 11 +- src/toil/test/sort/__init__.py | 2 +- src/toil/test/sort/restart_sort.py | 2 +- src/toil/test/sort/sort.py | 2 +- src/toil/test/sort/sortTest.py | 41 +-- src/toil/test/src/__init__.py | 2 +- src/toil/test/src/autoDeploymentTest.py | 9 +- src/toil/test/src/checkpointTest.py | 2 +- src/toil/test/src/deferredFunctionTest.py | 17 +- src/toil/test/src/dockerCheckTest.py | 17 +- src/toil/test/src/fileStoreTest.py | 64 ++-- src/toil/test/src/helloWorldTest.py | 2 +- src/toil/test/src/importExportFileTest.py | 3 +- src/toil/test/src/jobDescriptionTest.py | 3 +- src/toil/test/src/jobEncapsulationTest.py | 6 +- src/toil/test/src/jobFileStoreTest.py | 2 +- src/toil/test/src/jobServiceTest.py | 23 +- src/toil/test/src/jobTest.py | 8 +- src/toil/test/src/miscTests.py | 9 +- src/toil/test/src/promisedRequirementTest.py | 3 +- src/toil/test/src/promisesTest.py | 2 +- src/toil/test/src/realtimeLoggerTest.py | 2 +- src/toil/test/src/regularLogTest.py | 5 +- src/toil/test/src/resourceTest.py | 29 +- src/toil/test/src/restartDAGTest.py | 2 +- src/toil/test/src/resumabilityTest.py | 2 +- src/toil/test/src/retainTempDirTest.py | 2 +- src/toil/test/src/threadingTest.py | 1 - src/toil/test/src/toilContextManagerTest.py | 6 +- .../test/src/userDefinedJobArgTypeTest.py | 2 +- src/toil/test/src/workerTest.py | 2 +- src/toil/test/utils/__init__.py | 2 +- src/toil/test/utils/toilDebugTest.py | 2 +- src/toil/test/utils/utilsTest.py | 123 +++----- src/toil/test/wdl/builtinTest.py | 43 ++- src/toil/test/wdl/conftest.py | 2 +- src/toil/test/wdl/toilwdlTest.py | 38 +-- src/toil/toilState.py | 5 +- src/toil/utils/__init__.py | 36 --- src/toil/utils/toilClean.py | 36 +-- src/toil/utils/toilDebugFile.py | 54 ++-- src/toil/utils/toilDebugJob.py | 47 +-- src/toil/utils/toilDestroyCluster.py | 24 +- src/toil/utils/toilKill.py | 23 +- src/toil/utils/toilLaunchCluster.py | 121 ++++--- src/toil/utils/toilMain.py | 12 +- src/toil/utils/toilRsyncCluster.py | 26 +- src/toil/utils/toilSshCluster.py | 30 +- src/toil/utils/toilStats.py | 213 +++++-------- src/toil/utils/toilStatus.py | 61 ++-- src/toil/utils/toilUpdateEC2Instances.py | 24 +- src/toil/wdl/wdl_analysis.py | 19 +- src/toil/wdl/wdl_functions.py | 7 +- src/toil/wdl/wdl_synthesis.py | 16 +- src/toil/worker.py | 15 +- version_template.py | 3 +- 150 files changed, 1565 insertions(+), 1867 deletions(-) create mode 100644 requirements-dev.txt create mode 100644 src/toil/lib/resources.py delete mode 100644 src/toil/provisioners/.gitignore rename src/toil/{ => test}/batchSystems/parasolTestSupport.py (91%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e9fcd6482..8d6c65447e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,11 +1,14 @@ image: quay.io/vgteam/vg_ci_prebake:latest # Note that we must run in a privileged container for our internal Docker daemon to come up. +variables: + PYTHONIOENCODING: "utf-8" + DEBIAN_FRONTEND: "noninteractive" + before_script: - startdocker || true - docker info - cat /etc/hosts - - export PYTHONIOENCODING=utf-8 - mkdir -p ~/.kube && cp "$GITLAB_SECRET_FILE_KUBE_CONFIG" ~/.kube/config - mkdir -p ~/.aws && cp "$GITLAB_SECRET_FILE_AWS_CREDENTIALS" ~/.aws/credentials @@ -37,7 +40,7 @@ py36_appliance_build: stage: basic_tests script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq + - apt update && apt install -y tzdata jq - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && pip install pycparser && make develop extras=[all] && pip install htcondor awscli==1.16.272 # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS - python setup_gitlab_docker.py @@ -49,7 +52,7 @@ py37_batch_systems: stage: main_tests script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev + - apt update && apt install -y tzdata jq python3.7 python3.7-dev - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli==1.16.272 - make test tests=src/toil/test/batchSystems/batchSystemTest.py - make test tests=src/toil/test/mesos/MesosDataStructuresTest.py @@ -58,17 +61,18 @@ py37_cwl_v1.0: stage: main_tests script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev + - apt update && apt install -y tzdata jq python3.7 python3.7-dev - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[cwl,aws] - mypy --ignore-missing-imports --no-strict-optional $(pwd)/src/toil/cwl/cwltoil.py # make this a separate linting stage - python setup_gitlab_docker.py # login to increase the docker.io rate limit - make test tests=src/toil/test/cwl/cwlTest.py::CWLv10Test + - make test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testCwlexample py37_cwl_v1.1: stage: main_tests script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev + - apt update && apt install -y tzdata jq python3.7 python3.7-dev - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[cwl,aws] - python setup_gitlab_docker.py # login to increase the docker.io rate limit - make test tests=src/toil/test/cwl/cwlTest.py::CWLv11Test @@ -77,7 +81,7 @@ py37_cwl_v1.2: stage: main_tests script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev + - apt update && apt install -y tzdata jq python3.7 python3.7-dev - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[cwl,aws] - python setup_gitlab_docker.py # login to increase the docker.io rate limit - make test tests=src/toil/test/cwl/cwlTest.py::CWLv12Test @@ -86,7 +90,7 @@ py37_wdl: stage: main_tests script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev default-jre + - apt update && apt install -y tzdata jq python3.7 python3.7-dev default-jre - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] - which java &> /dev/null || { echo >&2 "Java is not installed. Install java to run these tests."; exit 1; } - make test tests=src/toil/test/wdl/toilwdlTest.py # needs java (default-jre) to run "GATK.jar" @@ -96,7 +100,7 @@ py37_jobstore_and_provisioning: stage: main_tests script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev + - apt update && apt install -y tzdata jq python3.7 python3.7-dev - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor - make test tests=src/toil/test/jobStores/jobStoreTest.py - make test tests=src/toil/test/sort/sortTest.py @@ -110,16 +114,17 @@ py37_main: stage: basic_tests script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev + - apt update && apt install -y tzdata jq python3.7 python3.7-dev - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor - make test tests=src/toil/test/src - make test tests=src/toil/test/utils +# - make test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testDocker py37_appliance_build: stage: basic_tests script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev + - apt update && apt install -y tzdata jq python3.7 python3.7-dev - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && pip install pycparser && make develop extras=[all] && pip install htcondor awscli==1.16.272 # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS - python setup_gitlab_docker.py @@ -129,7 +134,7 @@ py37_integration: stage: integration script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev + - apt update && apt install -y tzdata jq python3.7 python3.7-dev - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli==1.16.272 - export TOIL_TEST_INTEGRATIVE=True - export TOIL_AWS_KEYNAME=id_rsa @@ -143,7 +148,7 @@ py37_provisioner_integration: stage: integration script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata awscli jq python3.7 python3.7-dev + - apt update && apt install -y tzdata awscli jq python3.7 python3.7-dev - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli==1.16.272 - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa - echo $'Host *\n AddressFamily inet' > /root/.ssh/config @@ -155,6 +160,7 @@ py37_provisioner_integration: - make push_docker - make test tests=src/toil/test/sort/sortTest.py - make test tests=src/toil/test/provisioners/clusterScalerTest.py + - make test tests=src/toil/test/utils/utilsTest.py::UtilsTest::testAWSProvisionerUtils # - python -m pytest --duration=0 -s -r s src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSRestartTest::testAutoScaledCluster # - python -m pytest -s src/toil/test/provisioners/aws/awsProvisionerTest.py # - python -m pytest -s src/toil/test/provisioners/gceProvisionerTest.py # needs env vars set to run @@ -164,7 +170,7 @@ py38_main: stage: basic_tests script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.8 python3.8-dev + - apt update && apt install -y tzdata jq python3.8 python3.8-dev - virtualenv -p python3.8 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor - make test tests=src/toil/test/src - make test tests=src/toil/test/utils @@ -173,7 +179,7 @@ py38_appliance_build: stage: basic_tests script: - pwd - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.8 python3.8-dev + - apt update && apt install -y tzdata jq python3.8 python3.8-dev - virtualenv -p python3.8 venv && . venv/bin/activate && make prepare && pip install pycparser && make develop extras=[all] && pip install htcondor awscli==1.16.272 # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS - python setup_gitlab_docker.py @@ -184,7 +190,7 @@ py37_cactus_integration: stage: integration script: - set -e - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata awscli jq python3.7 python3.7-dev + - apt update && apt install -y tzdata awscli jq python3.7 python3.7-dev - virtualenv --system-site-packages --python python3.7 venv - . venv/bin/activate - pip install .[aws,kubernetes] diff --git a/Makefile b/Makefile index b6709adc3c..bfde924d50 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -113,7 +113,7 @@ clean_sdist: # We always claim to be Travis, so that local test runs will not skip Travis tests. test: check_venv check_build_reqs TRAVIS=true \ - python -m pytest --cov=toil --duration=0 -s -r s $(tests) + python -m pytest --cov=toil --durations=0 -s -r s $(tests) # This target will skip building docker and all docker based tests @@ -202,9 +202,7 @@ check_build_reqs: || ( printf "$(red)Build requirements are missing. Run 'make prepare' to install them.$(normal)\n" ; false ) prepare: check_venv - pip install mock==1.0.1 pytest==4.3.1 pytest-cov==2.6.1 stubserver==1.0.1 \ - pytest-timeout==1.3.3 setuptools==45.3.0 'sphinx>=2.4.4,<3' \ - cwltest mypy flake8 flake8-bugbear black isort pydocstyle + pip install -r requirements-dev.txt check_venv: @python -c 'import sys, os; sys.exit( int( 0 if "VIRTUAL_ENV" in os.environ else 1 ) )' \ @@ -231,10 +229,12 @@ PYSOURCES=$(shell find src -name '*.py') setup.py version_template.py # Linting and code style related targets ## sorting imports using isort: https://github.com/timothycrosley/isort sort_imports: $(PYSOURCES) - isort $^ + isort -m VERTICAL $^ + make format remove_unused_imports: $(PYSOURCES) autoflake --in-place --remove-all-unused-imports $^ + make format format: $(wildcard src/toil/cwl/*.py) black $^ diff --git a/docker/Dockerfile.py b/docker/Dockerfile.py index 45346e0160..a18d08e9e2 100644 --- a/docker/Dockerfile.py +++ b/docker/Dockerfile.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2020 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000000..6be4be7352 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,16 @@ +mock==4.0.3 +pytest==6.2.0 +pytest-cov==2.10.1 +pytest-timeout==1.4.2 +stubserver==1.1 +setuptools==51.0.0 +sphinx==3.3.1 +cwltest==2.0.20200626112502 +mypy==0.790 +flake8==3.8.4 +flake8-bugbear==20.11.1 +black +isort +pydocstyle +autoflake +isort diff --git a/setup.py b/setup.py index 299ffaf6fc..67a9e6d28b 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import imp +import os + +from tempfile import NamedTemporaryFile from setuptools import find_packages, setup -def runSetup(): +def run_setup(): """ Calls setup(). This function exists so the setup() invocation preceded more internal - functionality. The `version` module is imported dynamically by importVersion() below. + functionality. The `version` module is imported dynamically by import_version() below. """ boto = 'boto==2.48.0' boto3 = 'boto3>=1.7.50, <2.0' @@ -51,7 +55,6 @@ def runSetup(): addict, pytz, enlighten] - aws_reqs = [ boto, boto3, @@ -75,7 +78,6 @@ def runSetup(): pymesos, psutil] wdl_reqs = [] - # htcondor is not supported by apple # this is tricky to conditionally support in 'all' due @@ -89,7 +91,6 @@ def runSetup(): kubernetes_reqs + \ mesos_reqs - setup( name='toil', version=version.distVersion, @@ -135,7 +136,7 @@ def runSetup(): # Note that we intentionally include the top-level `test` package for # functionality like the @experimental and @integrative decorators: exclude=['*.test.*']), - package_data = { + package_data={ '': ['*.yml', 'cloud-config'], }, # Unfortunately, the names of the entry points are hard-coded elsewhere in the code base so @@ -152,44 +153,27 @@ def runSetup(): '_toil_kubernetes_executor = toil.batchSystems.kubernetes:executor [kubernetes]']}) -def importVersion(): - """ - Load and return the module object for src/toil/version.py, generating it from the template if - required. - """ - import imp - try: - # Attempt to load the template first. It only exists in a working copy cloned via git. - import version_template - except ImportError: - # If loading the template fails we must be in a unpacked source distribution and - # src/toil/version.py will already exist. - pass - else: +def import_version(): + """Return the module object for src/toil/version.py, generate from the template if required.""" + if not os.path.exists('src/toil/version.py'): # Use the template to generate src/toil/version.py - import errno - import os - from tempfile import NamedTemporaryFile - - new = version_template.expand_() - try: - with open('src/toil/version.py') as f: - old = f.read() - except IOError as e: - if e.errno == errno.ENOENT: - old = None - else: - raise + import version_template + with NamedTemporaryFile(mode='w', dir='src/toil', prefix='version.py.', delete=False) as f: + f.write(version_template.expand_()) + os.rename(f.name, 'src/toil/version.py') - if old != new: - with NamedTemporaryFile(mode='w', dir='src/toil', prefix='version.py.', delete=False) as f: - f.write(new) - os.rename(f.name, 'src/toil/version.py') # Unfortunately, we can't use a straight import here because that would also load the stuff - # defined in src/toil/__init__.py which imports modules from external dependencies that may + # defined in "src/toil/__init__.py" which imports modules from external dependencies that may # yet to be installed when setup.py is invoked. + # + # This is also the reason we cannot switch from the "deprecated" imp library + # and use: + # from importlib.machinery import SourceFileLoader + # return SourceFileLoader('toil.version', path='src/toil/version.py').load_module() + # + # Because SourceFileLoader will error and load "src/toil/__init__.py" . return imp.load_source('toil.version', 'src/toil/version.py') -version = importVersion() -runSetup() +version = import_version() +run_setup() diff --git a/setup_gitlab_docker.py b/setup_gitlab_docker.py index a756c75828..1faa83e22a 100644 --- a/setup_gitlab_docker.py +++ b/setup_gitlab_docker.py @@ -1,3 +1,16 @@ +# Copyright (C) 2015-2021 Regents of the University of California +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import json import os import subprocess diff --git a/setup_gitlab_ssh.py b/setup_gitlab_ssh.py index 8ce608c6f6..b326961c1a 100644 --- a/setup_gitlab_ssh.py +++ b/setup_gitlab_ssh.py @@ -1,3 +1,16 @@ +# Copyright (C) 2015-2021 Regents of the University of California +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import json import os diff --git a/src/toil/__init__.py b/src/toil/__init__.py index 0c93688629..de1c3fd49c 100644 --- a/src/toil/__init__.py +++ b/src/toil/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - import errno import logging import os @@ -430,7 +428,8 @@ def logProcessContext(config): try: from boto import provider - from botocore.credentials import (JSONFileCache, RefreshableCredentials, + from botocore.credentials import (JSONFileCache, + RefreshableCredentials, create_credential_resolver) from botocore.session import Session @@ -683,4 +682,3 @@ def _obtain_credentials_from_cache_or_boto3(self): except ImportError: pass - diff --git a/src/toil/batchSystems/__init__.py b/src/toil/batchSystems/__init__.py index ffd112c7aa..97c2989dad 100644 --- a/src/toil/batchSystems/__init__.py +++ b/src/toil/batchSystems/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - from functools import total_ordering @@ -22,7 +20,7 @@ class DeadlockException(Exception): resources to run the workflow """ def __init__(self, msg): - self.msg = "Deadlock encountered: " + msg + self.msg = f"Deadlock encountered: {msg}" super().__init__() def __str__(self): diff --git a/src/toil/batchSystems/abstractBatchSystem.py b/src/toil/batchSystems/abstractBatchSystem.py index 90b33634f8..b8c8e341c5 100644 --- a/src/toil/batchSystems/abstractBatchSystem.py +++ b/src/toil/batchSystems/abstractBatchSystem.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/batchSystems/abstractGridEngineBatchSystem.py b/src/toil/batchSystems/abstractGridEngineBatchSystem.py index 3162917811..32699a33ad 100644 --- a/src/toil/batchSystems/abstractGridEngineBatchSystem.py +++ b/src/toil/batchSystems/abstractGridEngineBatchSystem.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/batchSystems/gridengine.py b/src/toil/batchSystems/gridengine.py index 74cbde32d1..02ec8e3939 100644 --- a/src/toil/batchSystems/gridengine.py +++ b/src/toil/batchSystems/gridengine.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging import math import os @@ -19,8 +18,7 @@ from pipes import quote from toil.batchSystems import MemoryString -from toil.batchSystems.abstractGridEngineBatchSystem import \ - AbstractGridEngineBatchSystem +from toil.batchSystems.abstractGridEngineBatchSystem import AbstractGridEngineBatchSystem from toil.lib.misc import CalledProcessErrorStderr, call_command logger = logging.getLogger(__name__) diff --git a/src/toil/batchSystems/htcondor.py b/src/toil/batchSystems/htcondor.py index ad59f21d97..5e95bc86da 100644 --- a/src/toil/batchSystems/htcondor.py +++ b/src/toil/batchSystems/htcondor.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging import math import os @@ -20,8 +19,7 @@ import htcondor -from toil.batchSystems.abstractGridEngineBatchSystem import \ - AbstractGridEngineBatchSystem +from toil.batchSystems.abstractGridEngineBatchSystem import AbstractGridEngineBatchSystem logger = logging.getLogger(__name__) diff --git a/src/toil/batchSystems/kubernetes.py b/src/toil/batchSystems/kubernetes.py index f6d6f25de6..06a3bb6f18 100644 --- a/src/toil/batchSystems/kubernetes.py +++ b/src/toil/batchSystems/kubernetes.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Batch system for running Toil workflows on Kubernetes. @@ -40,14 +39,15 @@ from kubernetes.client.rest import ApiException from toil import applianceSelf -from toil.batchSystems.abstractBatchSystem import ( - EXIT_STATUS_UNAVAILABLE_VALUE, BatchJobExitReason, - BatchSystemCleanupSupport, UpdatedBatchJobInfo) +from toil.batchSystems.abstractBatchSystem import (EXIT_STATUS_UNAVAILABLE_VALUE, + BatchJobExitReason, + BatchSystemCleanupSupport, + UpdatedBatchJobInfo) from toil.common import Toil -from toil.lib.bioio import configureRootLogger, setLogLevel from toil.lib.humanize import human2bytes from toil.lib.retry import ErrorCondition, retry from toil.resource import Resource +from toil.statsAndLogging import configure_root_logger, set_log_level logger = logging.getLogger(__name__) retryable_kubernetes_errors = [urllib3.exceptions.MaxRetryError, @@ -1093,10 +1093,10 @@ def executor(): """ - configureRootLogger() - setLogLevel("DEBUG") + configure_root_logger() + set_log_level("DEBUG") logger.debug("Starting executor") - + # If we don't manage to run the child, what should our exit code be? exit_code = EXIT_STATUS_UNAVAILABLE_VALUE @@ -1146,5 +1146,3 @@ def executor(): Resource.cleanSystem() logger.debug('Shutting down') sys.exit(exit_code) - - diff --git a/src/toil/batchSystems/lsf.py b/src/toil/batchSystems/lsf.py index 0247a73982..84efc255fd 100644 --- a/src/toil/batchSystems/lsf.py +++ b/src/toil/batchSystems/lsf.py @@ -335,32 +335,29 @@ def obtainSystemConstants(cls): cpu_index = None mem_index = None for i in range(num_columns): - if items[i] == 'ncpus': - cpu_index = i - elif items[i] == 'maxmem': - mem_index = i + if items[i] == 'ncpus': + cpu_index = i + elif items[i] == 'maxmem': + mem_index = i if cpu_index is None or mem_index is None: - raise RuntimeError("lshosts command does not return ncpus or maxmem " - "columns") + raise RuntimeError("lshosts command does not return ncpus or maxmem columns") maxCPU = 0 maxMEM = MemoryString("0") for line in stdout.split('\n')[1:]: items = line.strip().split() - if not items: - continue - if len(items) < num_columns: - raise RuntimeError("lshosts output has a varying number of " - "columns") - if items[cpu_index] != '-' and int(items[cpu_index]) > int(maxCPU): - maxCPU = int(items[cpu_index]) - if (items[mem_index] != '-' and - MemoryString(items[mem_index]) > maxMEM): - maxMEM = MemoryString(items[mem_index]) + if items: + if len(items) < num_columns: + raise RuntimeError("lshosts output has a varying number of columns") + if items[cpu_index] != '-' and int(items[cpu_index]) > int(maxCPU): + maxCPU = int(items[cpu_index]) + if items[mem_index] != '-' and MemoryString(items[mem_index]) > maxMEM: + maxMEM = MemoryString(items[mem_index]) if maxCPU == 0 or maxMEM == MemoryString("0"): raise RuntimeError("lshosts returns null ncpus or maxmem info") + logger.debug("Got the maxMEM: {}".format(maxMEM)) logger.debug("Got the maxCPU: {}".format(maxCPU)) diff --git a/src/toil/batchSystems/mesos/__init__.py b/src/toil/batchSystems/mesos/__init__.py index 14c13c49bf..2da673b6a1 100644 --- a/src/toil/batchSystems/mesos/__init__.py +++ b/src/toil/batchSystems/mesos/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/batchSystems/mesos/batchSystem.py b/src/toil/batchSystems/mesos/batchSystem.py index f739874852..aa068777df 100644 --- a/src/toil/batchSystems/mesos/batchSystem.py +++ b/src/toil/batchSystems/mesos/batchSystem.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,9 +31,12 @@ from pymesos import MesosSchedulerDriver, Scheduler, decode_data, encode_data from toil import resolveEntryPoint -from toil.batchSystems.abstractBatchSystem import ( - EXIT_STATUS_UNAVAILABLE_VALUE, AbstractScalableBatchSystem, - BatchJobExitReason, BatchSystemLocalSupport, NodeInfo, UpdatedBatchJobInfo) +from toil.batchSystems.abstractBatchSystem import (EXIT_STATUS_UNAVAILABLE_VALUE, + AbstractScalableBatchSystem, + BatchJobExitReason, + BatchSystemLocalSupport, + NodeInfo, + UpdatedBatchJobInfo) from toil.batchSystems.mesos import JobQueue, MesosShape, TaskData, ToilJob from toil.lib.memoize import strict_bool diff --git a/src/toil/batchSystems/mesos/conftest.py b/src/toil/batchSystems/mesos/conftest.py index e904fef986..93a21d1e19 100644 --- a/src/toil/batchSystems/mesos/conftest.py +++ b/src/toil/batchSystems/mesos/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/batchSystems/mesos/executor.py b/src/toil/batchSystems/mesos/executor.py index 751c16e45e..afb46bf04f 100644 --- a/src/toil/batchSystems/mesos/executor.py +++ b/src/toil/batchSystems/mesos/executor.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import json import logging import os @@ -26,17 +25,17 @@ import threading import time import traceback - import addict import psutil + from pymesos import Executor, MesosExecutorDriver, decode_data, encode_data -from urllib2 import urlopen +from urllib.request import urlopen from toil.batchSystems.abstractBatchSystem import BatchSystemSupport -from toil.lib.bioio import configureRootLogger, setLogLevel from toil.lib.expando import Expando from toil.lib.threading import cpu_count from toil.resource import Resource +from toil.statsAndLogging import configure_root_logger, set_log_level log = logging.getLogger(__name__) @@ -233,8 +232,8 @@ def frameworkMessage(self, driver, message): def main(): - configureRootLogger() - setLogLevel("DEBUG") + configure_root_logger() + set_log_level("INFO") if not os.environ.get("MESOS_AGENT_ENDPOINT"): # Some Mesos setups in our tests somehow lack this variable. Provide a @@ -306,4 +305,3 @@ def patched_on_event(event): exit_value = 0 if (driver_result is None or driver_result == 'DRIVER_STOPPED') else 1 assert len(executor.runningTasks) == 0 sys.exit(exit_value) - diff --git a/src/toil/batchSystems/options.py b/src/toil/batchSystems/options.py index 41cfdc637a..30b36a0d7c 100644 --- a/src/toil/batchSystems/options.py +++ b/src/toil/batchSystems/options.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and import socket from contextlib import closing +from typing import Callable from toil.batchSystems.registry import (BATCH_SYSTEM_FACTORY_REGISTRY, - BATCH_SYSTEMS, DEFAULT_BATCH_SYSTEM) + BATCH_SYSTEMS, + DEFAULT_BATCH_SYSTEM) from toil.lib.threading import cpu_count @@ -39,23 +41,24 @@ def getPublicIP(): return '127.0.0.1' -def _parasolOptions(addOptionFn, config=None): - addOptionFn("--parasolCommand", dest="parasolCommand", default=None, - help="The name or path of the parasol program. Will be looked up on PATH " - "unless it starts with a slash. default=%s" % 'parasol') - addOptionFn("--parasolMaxBatches", dest="parasolMaxBatches", default=None, - help="Maximum number of job batches the Parasol batch is allowed to create. One " - "batch is created for jobs with a a unique set of resource requirements. " - "default=%i" % 1000) - - -def _singleMachineOptions(addOptionFn, config): - addOptionFn("--scale", dest="scale", default=None, - help=("A scaling factor to change the value of all submitted " - "tasks's submitted cores. Used in singleMachine batch " - "system. default=%s" % 1)) - if config.cwl: - addOptionFn( +def add_parasol_options(add_option: Callable): + add_option("--parasolCommand", dest="parasolCommand", default=None, + help="The name or path of the parasol program. Will be looked up on PATH " + "unless it starts with a slash. default=%s" % 'parasol') + add_option("--parasolMaxBatches", dest="parasolMaxBatches", default=None, + help="Maximum number of job batches the Parasol batch is allowed to create. One " + "batch is created for jobs with a a unique set of resource requirements. " + "default=%i" % 1000) + + +def add_single_machine_options(add_option: Callable, cwl: bool): + add_option("--scale", dest="scale", default=None, + help=("A scaling factor to change the value of all submitted " + "tasks's submitted cores. Used in singleMachine batch " + "system. default=%s" % 1)) + if cwl: + # TODO: Use both options; add as mutually exclusive groups + add_option( "--noLinkImports", dest="linkImports", default=True, action='store_false', help="When using a filesystem based job " "store, CWL input files are by default symlinked in. " @@ -64,7 +67,7 @@ def _singleMachineOptions(addOptionFn, config): "When not specified and as long as caching is enabled, Toil will " "protect the file automatically by changing the permissions to " "read-only.") - addOptionFn( + add_option( "--noMoveExports", dest="moveExports", default=True, action='store_false', help="When using a filesystem based job " "store, output files are by default moved to the output directory, " @@ -72,79 +75,76 @@ def _singleMachineOptions(addOptionFn, config): "Specifying this option instead copies the files into the output " "directory. Applies to filesystem-based job stores only.") else: - addOptionFn( + add_option( "--linkImports", dest="linkImports", default=False, action='store_true', help="When using Toil's importFile function " "for staging, input files are copied to the job store. Specifying " "this option saves space by sym-linking imported files. As long " "as caching is enabled Toil will protect the file " "automatically by changing the permissions to read-only.") - addOptionFn( + add_option( "--moveExports", dest="moveExports", default=False, action='store_true', help="When using Toil's exportFile function " "for staging, output files are copied to the output directory. Specifying " "this option saves space by moving exported files, and making a symlink to " "the exported file in the job store. Applies to filesystem-based job stores only.") -def _mesosOptions(addOptionFn, config=None): - addOptionFn("--mesosMaster", dest="mesosMasterAddress", default=getPublicIP() + ':5050', - help=("The host and port of the Mesos master separated by colon. (default: %(default)s)")) -def _kubernetesOptions(addOptionFn, config=None): - addOptionFn("--kubernetesHostPath", dest="kubernetesHostPath", default=None, - help=("Path on Kubernetes hosts to use as shared inter-pod temp directory (default: %(default)s)")) +def add_mesos_options(add_option: Callable): + add_option("--mesosMaster", dest="mesosMasterAddress", default=f'{getPublicIP()}:5050', + help="The host and port of the Mesos master separated by colon. (default: %(default)s)") + -# Built in batch systems that have options -_options = [ - _parasolOptions, - _singleMachineOptions, - _mesosOptions, - _kubernetesOptions - ] +def add_kubernetes_options(add_option: Callable): + add_option("--kubernetesHostPath", dest="kubernetesHostPath", default=None, + help="Path on Kubernetes hosts to use as shared inter-pod temp directory (default: %(default)s)") -def setOptions(config, setOption): - batch_system_factory = BATCH_SYSTEM_FACTORY_REGISTRY[config.batchSystem]() +def set_batchsystem_options(batch_system: str, setOption: Callable): + batch_system_factory = BATCH_SYSTEM_FACTORY_REGISTRY[batch_system]() batch_system_factory.setOptions(setOption) -def addOptions(addOptionFn, config): - addOptionFn("--batchSystem", dest="batchSystem", default=DEFAULT_BATCH_SYSTEM, choices=BATCH_SYSTEMS, - help=(f"The type of batch system to run the job(s) with, currently can be one " - f"of {', '.join(BATCH_SYSTEMS)}. default={DEFAULT_BATCH_SYSTEM}")) - addOptionFn("--disableHotDeployment", dest="disableAutoDeployment", - action='store_true', default=None, - help=("Hot-deployment was renamed to auto-deployment. Option now redirects to " - "--disableAutoDeployment. Left in for backwards compatibility.")) - addOptionFn("--disableAutoDeployment", dest="disableAutoDeployment", - action='store_true', default=None, - help=("Should auto-deployment of the user script be deactivated? If True, the user " - "script/package should be present at the same location on all workers. " - "default=false")) - localCores = cpu_count() - addOptionFn("--maxLocalJobs", default=localCores, - help="For batch systems that support a local queue for " - "housekeeping jobs (Mesos, GridEngine, htcondor, lsf, slurm, " - "torque), the maximum number of these housekeeping jobs to " - "run on the local system. " - "The default (equal to the number of cores) is a maximum of " - "{} concurrent local housekeeping jobs.".format(localCores)) - addOptionFn("--manualMemArgs", default=False, action='store_true', dest="manualMemArgs", - help="Do not add the default arguments: 'hv=MEMORY' & 'h_vmem=MEMORY' to " - "the qsub call, and instead rely on TOIL_GRIDGENGINE_ARGS to supply " - "alternative arguments. Requires that TOIL_GRIDGENGINE_ARGS be set.") - addOptionFn("--runCwlInternalJobsOnWorkers", dest="runCwlInternalJobsOnWorkers", - action='store_true', default=None, - help=("Whether to run CWL internal jobs (e.g. CWLScatter) on the worker nodes " +def add_all_batchsystem_options(add_option: Callable, config): + # TODO: Only add options for the system the user is specifying? + add_option("--batchSystem", dest="batchSystem", default=DEFAULT_BATCH_SYSTEM, choices=BATCH_SYSTEMS, + help=(f"The type of batch system to run the job(s) with, currently can be one " + f"of {', '.join(BATCH_SYSTEMS)}. default={DEFAULT_BATCH_SYSTEM}")) + add_option("--disableHotDeployment", dest="disableAutoDeployment", + action='store_true', default=None, + help=("Hot-deployment was renamed to auto-deployment. Option now redirects to " + "--disableAutoDeployment. Left in for backwards compatibility.")) + add_option("--disableAutoDeployment", dest="disableAutoDeployment", + action='store_true', default=None, + help=("Should auto-deployment of the user script be deactivated? If True, the user " + "script/package should be present at the same location on all workers. " + "default=false")) + add_option("--maxLocalJobs", default=cpu_count(), + help=f"For batch systems that support a local queue for " + f"housekeeping jobs (Mesos, GridEngine, htcondor, lsf, slurm, " + f"torque), the maximum number of these housekeeping jobs to " + f"run on the local system. " + f"The default (equal to the number of cores) is a maximum of " + f"{cpu_count()} concurrent local housekeeping jobs.") + add_option("--manualMemArgs", default=False, action='store_true', dest="manualMemArgs", + help="Do not add the default arguments: 'hv=MEMORY' & 'h_vmem=MEMORY' to " + "the qsub call, and instead rely on TOIL_GRIDGENGINE_ARGS to supply " + "alternative arguments. Requires that TOIL_GRIDGENGINE_ARGS be set.") + add_option("--runCwlInternalJobsOnWorkers", dest="runCwlInternalJobsOnWorkers", + action='store_true', default=None, + help="Whether to run CWL internal jobs (e.g. CWLScatter) on the worker nodes " "instead of the primary node. If false (default), then all such jobs are run on " "the primary node. Setting this to true can speed up the pipeline for very large " "workflows with many sub-workflows and/or scatters, provided that the worker " - "pool is large enough.")) + "pool is large enough.") - for o in _options: - o(addOptionFn, config) + add_parasol_options(add_option) + add_single_machine_options(add_option, config.cwl) + add_mesos_options(add_option) + add_kubernetes_options(add_option) -def setDefaultOptions(config): + +def set_batchsystem_config_defaults(config): """ Set default options for builtin batch systems. This is required if a Config object is not constructed from an Options object. @@ -166,9 +166,7 @@ def setDefaultOptions(config): config.moveExports = False # mesos - config.mesosMasterAddress = '%s:5050' % getPublicIP() + config.mesosMasterAddress = f'{getPublicIP()}:5050' # Kubernetes config.kubernetesHostPath = None - - diff --git a/src/toil/batchSystems/parasol.py b/src/toil/batchSystems/parasol.py index 1ee907d1ad..38a18e7362 100644 --- a/src/toil/batchSystems/parasol.py +++ b/src/toil/batchSystems/parasol.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ from toil.batchSystems.abstractBatchSystem import (BatchSystemSupport, UpdatedBatchJobInfo) from toil.common import Toil -from toil.lib.bioio import getTempFile +from toil.test import get_temp_file from toil.lib.iterables import concat logger = logging.getLogger(__name__) @@ -140,7 +140,7 @@ def issueBatchJob(self, jobDesc): try: results = self.resultsFiles[(truncatedMemory, jobDesc.cores)] except KeyError: - results = getTempFile(rootDir=self.parasolResultsDir) + results = get_temp_file(rootDir=self.parasolResultsDir) self.resultsFiles[(truncatedMemory, jobDesc.cores)] = results # Prefix the command with environment overrides, optionally looking them up from the @@ -366,4 +366,3 @@ def setOptions(cls, setOption): from toil.common import iC setOption("parasolCommand", None, None, 'parasol') setOption("parasolMaxBatches", int, iC(1), 10000) - diff --git a/src/toil/batchSystems/registry.py b/src/toil/batchSystems/registry.py index cbf83739c5..ab256459d3 100644 --- a/src/toil/batchSystems/registry.py +++ b/src/toil/batchSystems/registry.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/batchSystems/singleMachine.py b/src/toil/batchSystems/singleMachine.py index 66a43876a0..b48fb70450 100644 --- a/src/toil/batchSystems/singleMachine.py +++ b/src/toil/batchSystems/singleMachine.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging import math import os @@ -25,8 +24,9 @@ import toil from toil import worker as toil_worker -from toil.batchSystems.abstractBatchSystem import ( - EXIT_STATUS_UNAVAILABLE_VALUE, BatchSystemSupport, UpdatedBatchJobInfo) +from toil.batchSystems.abstractBatchSystem import (EXIT_STATUS_UNAVAILABLE_VALUE, + BatchSystemSupport, + UpdatedBatchJobInfo) from toil.common import Toil from toil.lib.threading import cpu_count @@ -582,7 +582,7 @@ def setOptions(cls, setOption): setOption("scale", default=1) -class Info(object): +class Info: """ Record for a running job. @@ -598,7 +598,7 @@ def __init__(self, startTime, popen, resources, killIntended): self.killIntended = killIntended -class ResourcePool(object): +class ResourcePool: """ Represents an integral amount of a resource (such as memory bytes). diff --git a/src/toil/batchSystems/slurm.py b/src/toil/batchSystems/slurm.py index df33b74a0b..0aaa46dd31 100644 --- a/src/toil/batchSystems/slurm.py +++ b/src/toil/batchSystems/slurm.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import math import os from pipes import quote -import math -from toil.lib.misc import call_command, CalledProcessErrorStderr from toil.batchSystems import MemoryString from toil.batchSystems.abstractGridEngineBatchSystem import AbstractGridEngineBatchSystem +from toil.lib.misc import CalledProcessErrorStderr, call_command logger = logging.getLogger(__name__) @@ -66,7 +66,7 @@ def submitJob(self, subLine): raise e def getJobExitCode(self, slurmJobID): - logger.debug("Getting exit code for slurm job %d", int(slurmJobID)) + logger.debug(f"Getting exit code for slurm job: {slurmJobID}") try: state, rc = self._getJobDetailsFromSacct(slurmJobID) @@ -128,7 +128,7 @@ def _getJobDetailsFromScontrol(self, slurmJobID): job = dict() for item in values: - logger.debug("%s output %s", args[0], item) + logger.debug(f"{args[0]} output {item}") # Output is in the form of many key=value pairs, multiple pairs on each line # and multiple lines in the output. Each pair is pulled out of each line and diff --git a/src/toil/batchSystems/torque.py b/src/toil/batchSystems/torque.py index 9a82993696..f5f1f7965f 100644 --- a/src/toil/batchSystems/torque.py +++ b/src/toil/batchSystems/torque.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging import math import os @@ -21,8 +20,8 @@ from pipes import quote from queue import Empty -from toil.batchSystems.abstractGridEngineBatchSystem import ( - AbstractGridEngineBatchSystem, UpdatedBatchJobInfo) +from toil.batchSystems.abstractGridEngineBatchSystem import (AbstractGridEngineBatchSystem, + UpdatedBatchJobInfo) from toil.lib.misc import CalledProcessErrorStderr, call_command logger = logging.getLogger(__name__) diff --git a/src/toil/common.py b/src/toil/common.py index 775e9b52ef..aa1724c964 100644 --- a/src/toil/common.py +++ b/src/toil/common.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,18 +25,20 @@ import requests from toil import logProcessContext, lookupEnvVar -from toil.batchSystems.options import addOptions as addBatchOptions -from toil.batchSystems.options import \ - setDefaultOptions as setDefaultBatchOptions -from toil.batchSystems.options import setOptions as setBatchOptions -from toil.lib.bioio import (addLoggingOptions, getLogLevelString, - setLoggingFromOptions) -from toil.lib.humanize import bytes2human +from toil.batchSystems.options import (add_all_batchsystem_options, + set_batchsystem_config_defaults, + set_batchsystem_options) +from toil.lib.humanize import bytes2human, human2bytes from toil.lib.retry import retry -from toil.provisioners import clusterFactory -from toil.provisioners.aws import checkValidNodeTypes, zoneToRegion +from toil.provisioners import (add_provisioner_options, + check_valid_node_types, + cluster_factory) +from toil.provisioners.aws import zone_to_region from toil.realtimeLogger import RealtimeLogger -from toil.version import dockerRegistry, dockerTag +from toil.statsAndLogging import (add_logging_options, + root_logger, + set_logging_from_options) +from toil.version import dockerRegistry, dockerTag, version # aim to pack autoscaling jobs within a 30 minute block before provisioning a new node defaultTargetTime = 1800 @@ -54,7 +56,7 @@ def __init__(self): finished sucessfully and its job store has been clean up.""" self.workflowAttemptNumber = None self.jobStore = None - self.logLevel = getLogLevelString() + self.logLevel = logging.getLevelName(root_logger.getEffectiveLevel()) self.workDir = None self.noStdOutErr = False self.stats = False @@ -69,12 +71,12 @@ def __init__(self): self.restart = False # Batch system options - setDefaultBatchOptions(self) + set_batchsystem_config_defaults(self) # Autoscaling options self.provisioner = None self.nodeTypes = [] - checkValidNodeTypes(self.provisioner, self.nodeTypes) + check_valid_node_types(self.provisioner, self.nodeTypes) self.nodeOptions = None self.minNodes = None self.maxNodes = [10] @@ -137,24 +139,18 @@ def __init__(self): def setOptions(self, options): """Creates a config object from the options object.""" - from toil.lib.humanize import human2bytes - def setOption(varName, parsingFn=None, checkFn=None, default=None): - # If options object has the option "varName" specified - # then set the "varName" attrib to this value in the config object - x = getattr(options, varName, None) - if x is None: - x = default - - if x is not None: + def setOption(option_name, parsingFn=None, checkFn=None, default=None): + option_value = getattr(options, option_name, default) + + if option_value is not None: if parsingFn is not None: - x = parsingFn(x) + option_value = parsingFn(option_value) if checkFn is not None: try: - checkFn(x) + checkFn(option_value) except AssertionError: - raise RuntimeError("The %s option has an invalid value: %s" - % (varName, x)) - setattr(self, varName, x) + raise RuntimeError(f"The {option_name} option has an invalid value: {option_value}") + setattr(self, option_name, option_value) # Function to parse integer from string expressed in different formats h2b = lambda x: human2bytes(str(x)) @@ -201,7 +197,7 @@ def parseIntList(s: str): # Batch system options setOption("batchSystem") - setBatchOptions(self, setOption) + set_batchsystem_options(self.batchSystem, setOption) setOption("disableAutoDeployment") setOption("scale", float, fC(0.0)) setOption("parasolCommand") @@ -220,18 +216,15 @@ def parseIntList(s: str): setOption("maxNodes", parseIntList) setOption("targetTime", int) if self.targetTime <= 0: - raise RuntimeError('targetTime (%s) must be a positive integer!' - '' % self.targetTime) + raise RuntimeError(f'targetTime ({self.targetTime}) must be a positive integer!') setOption("betaInertia", float) if not 0.0 <= self.betaInertia <= 0.9: - raise RuntimeError('betaInertia (%f) must be between 0.0 and 0.9!' - '' % self.betaInertia) + raise RuntimeError(f'betaInertia ({self.betaInertia}) must be between 0.0 and 0.9!') setOption("scaleInterval", float) setOption("metrics") setOption("preemptableCompensation", float) if not 0.0 <= self.preemptableCompensation <= 1.0: - raise RuntimeError('preemptableCompensation (%f) must be between 0.0 and 1.0!' - '' % self.preemptableCompensation) + raise RuntimeError(f'preemptableCompensation ({self.preemptableCompensation}) must be between 0.0 and 1.0!') setOption("nodeStorage", int) def checkNodeStorageOverrides(nodeStorageOverrides): @@ -307,29 +300,56 @@ def __hash__(self): return self.__dict__.__hash__() -jobStoreLocatorHelp = ("A job store holds persistent information about the jobs and files in a " - "workflow. If the workflow is run with a distributed batch system, the job " - "store must be accessible by all worker nodes. Depending on the desired " - "job store implementation, the location should be formatted according to " - "one of the following schemes:\n\n" - "file: where points to a directory on the file systen\n\n" - "aws:: where is the name of an AWS region like " - "us-west-2 and will be prepended to the names of any top-level " - "AWS resources in use by job store, e.g. S3 buckets.\n\n " - "google:: TODO: explain\n\n" - "For backwards compatibility, you may also specify ./foo (equivalent to " - "file:./foo or just file:foo) or /bar (equivalent to file:/bar).") +JOBSTORE_HELP = ("The location of the job store for the workflow. " + "A job store holds persistent information about the jobs, stats, and files in a " + "workflow. If the workflow is run with a distributed batch system, the job " + "store must be accessible by all worker nodes. Depending on the desired " + "job store implementation, the location should be formatted according to " + "one of the following schemes:\n\n" + "file: where points to a directory on the file systen\n\n" + "aws:: where is the name of an AWS region like " + "us-west-2 and will be prepended to the names of any top-level " + "AWS resources in use by job store, e.g. S3 buckets.\n\n " + "google:: TODO: explain\n\n" + "For backwards compatibility, you may also specify ./foo (equivalent to " + "file:./foo or just file:foo) or /bar (equivalent to file:/bar).") + + +def parser_with_common_options(provisioner_options=False, jobstore_option=True): + parser = ArgumentParser() + + if provisioner_options: + add_provisioner_options(parser) + + if jobstore_option: + parser.add_argument('jobStore', type=str, help=JOBSTORE_HELP) + # always add these + add_logging_options(parser) + parser.add_argument("--version", action='version', version=version) + parser.add_argument("--tempDirRoot", dest="tempDirRoot", type=str, default=tempfile.gettempdir(), + help="Path to where temporary directory containing all temp files are created, " + "by default generates a fresh tmp dir with 'tempfile.gettempdir()'.") + return parser + + +def addOptions(parser: ArgumentParser, config: Config = Config()): + if not isinstance(parser, ArgumentParser): + raise ValueError(f"Unanticipated class: {parser.__class__}. Must be: argparse.ArgumentParser.") + + def addGroupFn(group_title, group_description): + return parser.add_argument_group(group_title, group_description).add_argument + + add_logging_options(parser) + parser.register("type", "bool", parseBool) # Custom type for arg=True/False. -def _addOptions(addGroupFn, config): # # Core options # - addOptionFn = addGroupFn("toil core options", + addOptionFn = addGroupFn("Toil core options.", "Options to specify the location of the Toil workflow and turn on " "stats collation about the performance of jobs.") - addOptionFn('jobStore', type=str, - help="The location of the job store for the workflow. " + jobStoreLocatorHelp) + addOptionFn('jobStore', type=str, help=JOBSTORE_HELP) addOptionFn("--workDir", dest="workDir", default=None, help="Absolute path to directory where temporary files generated during the Toil " "run should be placed. Standard output and error from batch system jobs " @@ -379,14 +399,13 @@ def _addOptions(addGroupFn, config): # # Batch system options # - addOptionFn = addGroupFn("toil options for specifying the batch system", "Allows the specification of the batch system, and arguments to the " "batch system/big batch system (see below).") addOptionFn("--statePollingWait", dest="statePollingWait", default=1, type=int, help=("Time, in seconds, to wait before doing a scheduler query for job state. " "Return cached results if within the waiting period.")) - addBatchOptions(addOptionFn, config) + add_all_batchsystem_options(addOptionFn, config) # # Auto scaling options @@ -651,23 +670,6 @@ def parseBool(val): else: raise RuntimeError("Could not interpret \"%s\" as a boolean value" % val) -def addOptions(parser, config=Config()): - """ - Adds toil options to a parser object, either optparse or argparse. - """ - # Wrapper function that allows toil to be used with both the optparse and - # argparse option parsing modules - addLoggingOptions(parser) # This adds the logging stuff. - if isinstance(parser, ArgumentParser): - def addGroup(headingString, bodyString): - return parser.add_argument_group(headingString, bodyString).add_argument - - parser.register("type", "bool", parseBool) # Custom type for arg=True/False. - _addOptions(addGroup, config) - else: - raise RuntimeError("Unanticipated class passed to addOptions(), %s. Expecting " - "argparse.ArgumentParser" % parser.__class__) - def getNodeID(): """ @@ -761,7 +763,7 @@ def __enter__(self): consolidate the derived configuration with the one from the previous invocation of the workflow. """ - setLoggingFromOptions(self.options) + set_logging_from_options(self.options) config = Config() config.setOptions(self.options) jobStore = self.getJobStore(config.jobStore) @@ -789,11 +791,11 @@ def __exit__(self, exc_type, exc_val, exc_tb): if (exc_type is not None and self.config.clean == "onError" or exc_type is None and self.config.clean == "onSuccess" or self.config.clean == "always"): - - try: + + try: if self.config.restart and not self._inRestart: pass - else: + else: self._jobStore.destroy() logger.info("Successfully deleted the job store: %s" % str(self._jobStore)) except: @@ -885,12 +887,12 @@ def _setProvisioner(self): if self.config.provisioner is None: self._provisioner = None else: - self._provisioner = clusterFactory(provisioner=self.config.provisioner, - clusterName=None, - zone=None, # read from instance meta-data - nodeStorage=self.config.nodeStorage, - nodeStorageOverrides=self.config.nodeStorageOverrides, - sseKey=self.config.sseKey) + self._provisioner = cluster_factory(provisioner=self.config.provisioner, + clusterName=None, + zone=None, # read from instance meta-data + nodeStorage=self.config.nodeStorage, + nodeStorageOverrides=self.config.nodeStorageOverrides, + sseKey=self.config.sseKey) self._provisioner.setAutoscaledNodeTypes(self.config.nodeTypes) @classmethod @@ -1187,7 +1189,7 @@ def __init__(self, provisioner=None): if provisioner._zone is not None: if provisioner.cloud == 'aws': # Remove AZ name - region = zoneToRegion(provisioner._zone) + region = zone_to_region(provisioner._zone) else: region = provisioner._zone @@ -1330,9 +1332,6 @@ def shutdown(self): self.nodeExporterProc.kill() -# Nested functions can't have doctests so we have to make this global - - def parseSetEnv(l): """ Parses a list of strings of the form "NAME=VALUE" or just "NAME" into a dictionary. Strings @@ -1396,7 +1395,7 @@ def cacheDirName(workflowID): """ :return: Name of the cache directory. """ - return 'cache-' + workflowID + return f'cache-{workflowID}' def getDirSizeRecursively(dirPath): @@ -1422,7 +1421,7 @@ def getDirSizeRecursively(dirPath): # The call: 'du -s /some/path' should give the number of 512-byte blocks # allocated with the environment variable: BLOCKSIZE='512' set, and we # multiply this by 512 to return the filesize in bytes. - + try: return int(subprocess.check_output(['du', '-s', dirPath], env=dict(os.environ, BLOCKSIZE='512')).decode('utf-8').split()[0]) * 512 @@ -1445,6 +1444,7 @@ def getFileSystemSize(dirPath): diskSize = diskStats.f_frsize * diskStats.f_blocks return freeSpace, diskSize + def safeUnpickleFromStream(stream): string = stream.read() return pickle.loads(string) diff --git a/src/toil/cwl/__init__.py b/src/toil/cwl/__init__.py index 8b13789179..e69de29bb2 100644 --- a/src/toil/cwl/__init__.py +++ b/src/toil/cwl/__init__.py @@ -1 +0,0 @@ - diff --git a/src/toil/cwl/conftest.py b/src/toil/cwl/conftest.py index 183fac2a3f..121dc35e71 100644 --- a/src/toil/cwl/conftest.py +++ b/src/toil/cwl/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py index af1c26069e..c36586fa5b 100755 --- a/src/toil/cwl/cwltoil.py +++ b/src/toil/cwl/cwltoil.py @@ -1,6 +1,6 @@ """Implemented support for Common Workflow Language (CWL) for Toil.""" # Copyright (C) 2015 Curoverse, Inc -# Copyright (C) 2016-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # Copyright (C) 2019-2020 Seven Bridges # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -55,6 +55,7 @@ import cwltool.provenance import cwltool.resolver import cwltool.stdfsaccess +import schema_salad.ref_resolver from cwltool.loghandler import _logger as cwllogger from cwltool.loghandler import defaultStreamHandler from cwltool.mutation import MutationManager @@ -72,8 +73,8 @@ get_container_from_software_requirements, ) from cwltool.utils import ( - CWLOutputAtomType, CWLObjectType, + CWLOutputAtomType, adjustDirObjs, adjustFileObjs, aslist, @@ -86,7 +87,6 @@ from schema_salad import validate from schema_salad.schema import Names from schema_salad.sourceline import SourceLine -import schema_salad.ref_resolver from toil.common import Config, Toil, addOptions from toil.fileStores import FileID diff --git a/src/toil/deferred.py b/src/toil/deferred.py index b64f341955..2025b22575 100644 --- a/src/toil/deferred.py +++ b/src/toil/deferred.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2019 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import fcntl import logging import os @@ -27,6 +26,7 @@ logger = logging.getLogger(__name__) + class DeferredFunction(namedtuple('DeferredFunction', 'function args kwargs name module')): """ >>> from collections import defaultdict @@ -341,8 +341,3 @@ def _runOrphanedDeferredFunctions(self): # Now close it. This closes the backing file descriptor. See # fileObj.close() - - - - - diff --git a/src/toil/fileStores/__init__.py b/src/toil/fileStores/__init__.py index b29d12ca61..a46411da14 100644 --- a/src/toil/fileStores/__init__.py +++ b/src/toil/fileStores/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2019 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import os -__all__ = ['fileStore', 'nonCachingFileStore', 'cachingFileStore', 'FileID'] + +def make_public_dir(dirName: str) -> str: + """Makes a given subdirectory if it doesn't already exist, making sure it is public.""" + if not os.path.exists(dirName): + os.mkdir(dirName) + os.chmod(dirName, 0o777) + return dirName class FileID(str): diff --git a/src/toil/fileStores/abstractFileStore.py b/src/toil/fileStores/abstractFileStore.py index e4e5acce85..fb14ddfd88 100644 --- a/src/toil/fileStores/abstractFileStore.py +++ b/src/toil/fileStores/abstractFileStore.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -511,6 +511,3 @@ def shutdown(cls, dir_): directories from the node. """ raise NotImplementedError() - - - diff --git a/src/toil/fileStores/cachingFileStore.py b/src/toil/fileStores/cachingFileStore.py index 1d7b78dd08..98e7f7673b 100644 --- a/src/toil/fileStores/cachingFileStore.py +++ b/src/toil/fileStores/cachingFileStore.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,9 +25,8 @@ from contextlib import contextmanager from toil.common import cacheDirName, getDirSizeRecursively, getFileSystemSize -from toil.fileStores import FileID +from toil.fileStores import FileID, make_public_dir from toil.fileStores.abstractFileStore import AbstractFileStore -from toil.lib.bioio import makePublicDir from toil.lib.humanize import bytes2human from toil.lib.misc import atomic_copy, atomic_copyobj, robust_rmtree from toil.lib.retry import ErrorCondition, retry @@ -975,7 +974,7 @@ def open(self, job): # Create a working directory for the job startingDir = os.getcwd() # Move self.localTempDir from the worker directory set up in __init__ to a per-job directory. - self.localTempDir = makePublicDir(os.path.join(self.localTempDir, str(uuid.uuid4()))) + self.localTempDir = make_public_dir(os.path.join(self.localTempDir, str(uuid.uuid4()))) # Check the status of all jobs on this node. If there are jobs that started and died before # cleaning up their presence from the database, clean them up ourselves. self._removeDeadJobs(self.workDir, self.con) diff --git a/src/toil/fileStores/nonCachingFileStore.py b/src/toil/fileStores/nonCachingFileStore.py index 85e4f8b51e..a612fec679 100644 --- a/src/toil/fileStores/nonCachingFileStore.py +++ b/src/toil/fileStores/nonCachingFileStore.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - import errno import fcntl import logging import os -import sys import uuid from collections import defaultdict from contextlib import contextmanager @@ -25,19 +22,14 @@ import dill from toil.common import getDirSizeRecursively, getFileSystemSize -from toil.fileStores import FileID +from toil.fileStores import FileID, make_public_dir from toil.fileStores.abstractFileStore import AbstractFileStore -from toil.lib.bioio import makePublicDir from toil.lib.humanize import bytes2human from toil.lib.misc import robust_rmtree from toil.lib.threading import get_process_name, process_name_exists logger = logging.getLogger(__name__) -if sys.version_info[0] < 3: - # Define a usable FileNotFoundError as will be raised by os.oprn on a - # nonexistent parent directory. - FileNotFoundError = OSError class NonCachingFileStore(AbstractFileStore): def __init__(self, jobStore, jobDesc, localTempDir, waitForPreviousCommit): @@ -50,7 +42,7 @@ def __init__(self, jobStore, jobDesc, localTempDir, waitForPreviousCommit): def open(self, job): jobReqs = job.disk startingDir = os.getcwd() - self.localTempDir = makePublicDir(os.path.join(self.localTempDir, str(uuid.uuid4()))) + self.localTempDir = make_public_dir(os.path.join(self.localTempDir, str(uuid.uuid4()))) self._removeDeadJobs(self.workDir) self.jobStateFile = self._createJobStateFile() freeSpace, diskSize = getFileSystemSize(self.localTempDir) @@ -272,4 +264,3 @@ def shutdown(cls, dir_): :param dir_: The workflow directory that will contain all the individual worker directories. """ cls._removeDeadJobs(dir_, batchSystemShutdown=True) - diff --git a/src/toil/job.py b/src/toil/job.py index 71e2cec486..8a21eefc92 100644 --- a/src/toil/job.py +++ b/src/toil/job.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - import collections import copy import importlib @@ -35,13 +33,15 @@ from toil.common import Config, Toil, addOptions, safeUnpickleFromStream from toil.deferred import DeferredFunction from toil.fileStores import FileID -from toil.lib.bioio import (getTotalCpuTime, getTotalCpuTimeAndMemoryUsage, - setLoggingFromOptions) from toil.lib.expando import Expando from toil.lib.humanize import human2bytes +from toil.lib.resources import (get_total_cpu_time, + get_total_cpu_time_and_memory_usage) from toil.resource import ModuleDescriptor +from toil.statsAndLogging import set_logging_from_options + +logger = logging.getLogger(__name__) -logger = logging.getLogger( __name__ ) class JobPromiseConstraintError(RuntimeError): """ @@ -64,7 +64,7 @@ def __init__(self, promisingJob, recipientJob=None): else: # Write a full error message super().__init__(f"Job {promisingJob.description} cannot promise its return value to non-successor {recipientJob.description}") - + class TemporaryID: """ @@ -100,6 +100,7 @@ def __eq__(self, other): def __ne__(self, other): return not isinstance(other, TemporaryID) or self._value != other._value + class Requirer: """ Base class implementing the storage and presentation of requirements for @@ -755,7 +756,7 @@ def setupJobAfterFailure(self, exitReason=None): def getLogFileHandle(self, jobStore): """ Returns a context manager that yields a file handle to the log file. - + Assumes logJobStoreFileID is set. """ return jobStore.readFileStream(self.logJobStoreFileID) @@ -1726,7 +1727,7 @@ def startToil(job, options): :return: The return value of the root job's run function. :rtype: Any """ - setLoggingFromOptions(options) + set_logging_from_options(options) with Toil(options) as toil: if not options.restart: return toil.start(job) @@ -2289,7 +2290,7 @@ def _executor(self, stats, fileStore): """ if stats is not None: startTime = time.time() - startClock = getTotalCpuTime() + startClock = get_total_cpu_time() baseDir = os.getcwd() yield @@ -2313,7 +2314,7 @@ def _executor(self, stats, fileStore): os.chdir(baseDir) # Finish up the stats if stats is not None: - totalCpuTime, totalMemoryUsage = getTotalCpuTimeAndMemoryUsage() + totalCpuTime, totalMemoryUsage = get_total_cpu_time_and_memory_usage() stats.jobs.append( Expando( time=str(time.time() - startTime), @@ -2655,6 +2656,7 @@ def getUserScript(self): assert self.encapsulatedJob is not None return self.encapsulatedJob.getUserScript() + class ServiceHostJob(Job): """ Job that runs a service. Used internally by Toil. Users should subclass Service instead of using this. @@ -2794,7 +2796,7 @@ def getUserScript(self): return self.serviceModule -class Promise(): +class Promise: """ References a return value from a :meth:`toil.job.Job.run` or :meth:`toil.job.Job.Service.start` method as a *promise* before the method itself is run. @@ -2866,7 +2868,7 @@ def _resolve(cls, jobStoreLocator, jobStoreFileID): return value -class PromisedRequirement(): +class PromisedRequirement: def __init__(self, valueOrCallable, *args): """ Class for dynamically allocating job function resource requirements involving @@ -2924,7 +2926,7 @@ def convertPromises(kwargs): return False -class UnfulfilledPromiseSentinel(): +class UnfulfilledPromiseSentinel: """This should be overwritten by a proper promised value. Throws an exception when unpickled.""" def __init__(self, fulfillingJobName, unpickled): diff --git a/src/toil/jobStores/abstractJobStore.py b/src/toil/jobStores/abstractJobStore.py index b78490a7de..42ee42b53b 100644 --- a/src/toil/jobStores/abstractJobStore.py +++ b/src/toil/jobStores/abstractJobStore.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,7 +27,8 @@ from toil.common import safeUnpickleFromStream from toil.fileStores import FileID -from toil.job import (CheckpointJobDescription, JobException, +from toil.job import (CheckpointJobDescription, + JobException, ServiceJobDescription) from toil.lib.memoize import memoize from toil.lib.misc import WriteWatchingStream @@ -104,7 +105,7 @@ class AbstractJobStore(ABC): """ Represents the physical storage for the jobs and files in a Toil workflow. - JobStores are responsible for storing :class:`toil.job.JobDescription`s + JobStores are responsible for storing :class:`toil.job.JobDescription` (which relate jobs to each other) and files. Actual :class:`toil.job.Job` objects are stored in files, referenced by @@ -112,8 +113,7 @@ class AbstractJobStore(ABC): in JobDescriptions and not full, executable Jobs. To actually get ahold of a :class:`toil.job.Job`, use - :meth:`toil.job.Job.loadJob` with a JobStore and the relevant - JobDescription. + :meth:`toil.job.Job.loadJob` with a JobStore and the relevant JobDescription. """ def __init__(self): diff --git a/src/toil/jobStores/aws/jobStore.py b/src/toil/jobStores/aws/jobStore.py index dc5837c13b..d16ba28302 100644 --- a/src/toil/jobStores/aws/jobStore.py +++ b/src/toil/jobStores/aws/jobStore.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - import base64 import hashlib import itertools @@ -37,21 +35,29 @@ import toil.lib.encryption as encryption from toil.fileStores import FileID -from toil.jobStores.abstractJobStore import ( - AbstractJobStore, ConcurrentFileModificationException, - JobStoreExistsException, NoSuchFileException, NoSuchJobException, - NoSuchJobStoreException) -from toil.jobStores.aws.utils import (SDBHelper, bucket_location_to_region, - chunkedFileUpload, copyKeyMultipart, +from toil.jobStores.abstractJobStore import (AbstractJobStore, + ConcurrentFileModificationException, + JobStoreExistsException, + NoSuchFileException, + NoSuchJobException, + NoSuchJobStoreException) +from toil.jobStores.aws.utils import (SDBHelper, + bucket_location_to_region, + chunkedFileUpload, + copyKeyMultipart, fileSizeAndTime, monkeyPatchSdbConnection, no_such_sdb_domain, - region_to_bucket_location, retry_s3, - retry_sdb, retryable_s3_errors, - sdb_unavailable, uploadFromPath) -from toil.jobStores.utils import (ReadablePipe, ReadableTransformingPipe, + region_to_bucket_location, + retry_s3, + retry_sdb, + retryable_s3_errors, + sdb_unavailable, + uploadFromPath) +from toil.jobStores.utils import (ReadablePipe, + ReadableTransformingPipe, WritablePipe) -from toil.lib.compatibility import compat_bytes, compat_plain +from toil.lib.compatibility import compat_bytes, compat_bytes from toil.lib.ec2nodes import EC2Regions from toil.lib.exceptions import panic from toil.lib.memoize import strict_bool @@ -65,12 +71,12 @@ boto3_session = boto3.Session(botocore_session=botocore_session) s3_boto3_resource = boto3_session.resource('s3') s3_boto3_client = boto3_session.client('s3') -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) + class ChecksumError(Exception): - """ - Raised when a download from AWS does not contain the correct data. - """ + """Raised when a download from AWS does not contain the correct data.""" + class AWSJobStore(AbstractJobStore): """ @@ -118,8 +124,8 @@ def __init__(self, locator, partSize=50 << 20): if '--' in namePrefix: raise ValueError("Invalid name prefix '%s'. Name prefixes may not contain " "%s." % (namePrefix, self.nameSeparator)) - log.debug("Instantiating %s for region %s and name prefix '%s'", - self.__class__, region, namePrefix) + logger.debug("Instantiating %s for region %s and name prefix '%s'", + self.__class__, region, namePrefix) self.locator = locator self.region = region self.namePrefix = namePrefix @@ -137,7 +143,7 @@ def initialize(self, config): try: self._bind(create=True) except: - with panic(log): + with panic(logger): self.destroy() else: super(AWSJobStore, self).initialize(config) @@ -251,7 +257,7 @@ def _awsJobFromItem(self, item): assert self.fileExists(item["overlargeID"]) # This is an overlarge job, download the actual attributes # from the file store - log.debug("Loading overlarge job from S3.") + logger.debug("Loading overlarge job from S3.") with self.readFileStream(item["overlargeID"]) as fh: binary = fh.read() else: @@ -293,8 +299,8 @@ def batch(self): def assignID(self, jobDescription): jobStoreID = self._newJobID() - log.debug("Assigning ID to job %s for '%s'", - jobStoreID, '' if jobDescription.command is None else jobDescription.command) + logger.debug("Assigning ID to job %s for '%s'", + jobStoreID, '' if jobDescription.command is None else jobDescription.command) jobDescription.jobStoreID = jobStoreID def create(self, jobDescription): @@ -333,11 +339,11 @@ def load(self, jobStoreID): job = self._awsJobFromItem(item) if job is None: raise NoSuchJobException(jobStoreID) - log.debug("Loaded job %s", jobStoreID) + logger.debug("Loaded job %s", jobStoreID) return job def update(self, jobDescription): - log.debug("Updating job %s", jobDescription.jobStoreID) + logger.debug("Updating job %s", jobDescription.jobStoreID) item = self._awsJobToItem(jobDescription) for attempt in retry_sdb(): with attempt: @@ -347,7 +353,7 @@ def update(self, jobDescription): def delete(self, jobStoreID): # remove job and replace with jobStoreId. - log.debug("Deleting job %s", jobStoreID) + logger.debug("Deleting job %s", jobStoreID) # If the job is overlarge, delete its file from the filestore item = None @@ -356,7 +362,7 @@ def delete(self, jobStoreID): item = self.jobsDomain.get_attributes(compat_bytes(jobStoreID), consistent_read=True) self._checkItem(item) if item["overlargeID"]: - log.debug("Deleting job from filestore") + logger.debug("Deleting job from filestore") self.deleteFile(item["overlargeID"]) for attempt in retry_sdb(): with attempt: @@ -370,7 +376,7 @@ def delete(self, jobStoreID): self.filesDomain.name, jobStoreID))) assert items is not None if items: - log.debug("Deleting %d file(s) associated with job %s", len(items), jobStoreID) + logger.debug("Deleting %d file(s) associated with job %s", len(items), jobStoreID) n = self.itemsPerBatchDelete batches = [items[i:i + n] for i in range(0, len(items), n)] for batch in batches: @@ -393,7 +399,7 @@ def getEmptyFileStoreID(self, jobStoreID=None, cleanup=False, basename=None): # Empty pass info.save() - log.debug("Created %r.", info) + logger.debug("Created %r.", info) return info.fileID def _importFile(self, otherCls, url, sharedFileName=None, hardlink=False): @@ -458,11 +464,11 @@ def _writeToUrl(cls, readable, url): except: canDetermineSize = False if canDetermineSize and fileSize > (5 * 1000 * 1000): # only use multipart when file is above 5 mb - log.debug("Uploading %s with size %s, will use multipart uploading", dstKey.name, fileSize) + logger.debug("Uploading %s with size %s, will use multipart uploading", dstKey.name, fileSize) chunkedFileUpload(readable=readable, bucket=dstKey.bucket, fileID=dstKey.name, file_size=fileSize) else: # we either don't know the size, or the size is small - log.debug("Can not use multipart uploading for %s, uploading whole file at once", dstKey.name) + logger.debug("Can not use multipart uploading for %s, uploading whole file at once", dstKey.name) dstKey.set_contents_from_string(readable.read()) finally: dstKey.bucket.connection.close() @@ -527,7 +533,7 @@ def writeFile(self, localFilePath, jobStoreID=None, cleanup=False): info = self.FileInfo.create(jobStoreID if cleanup else None) info.upload(localFilePath, not self.config.disableJobStoreChecksumVerification) info.save() - log.debug("Wrote %r of from %r", info, localFilePath) + logger.debug("Wrote %r of from %r", info, localFilePath) return info.fileID @contextmanager @@ -536,7 +542,7 @@ def writeFileStream(self, jobStoreID=None, cleanup=False, basename=None): with info.uploadStream() as writable: yield writable, info.fileID info.save() - log.debug("Wrote %r.", info) + logger.debug("Wrote %r.", info) @contextmanager def writeSharedFileStream(self, sharedFileName, isProtected=None): @@ -547,13 +553,13 @@ def writeSharedFileStream(self, sharedFileName, isProtected=None): with info.uploadStream() as writable: yield writable info.save() - log.debug("Wrote %r for shared file %r.", info, sharedFileName) + logger.debug("Wrote %r for shared file %r.", info, sharedFileName) def updateFile(self, jobStoreFileID, localFilePath): info = self.FileInfo.loadOrFail(jobStoreFileID) info.upload(localFilePath, not self.config.disableJobStoreChecksumVerification) info.save() - log.debug("Wrote %r from path %r.", info, localFilePath) + logger.debug("Wrote %r from path %r.", info, localFilePath) @contextmanager def updateFileStream(self, jobStoreFileID): @@ -561,7 +567,7 @@ def updateFileStream(self, jobStoreFileID): with info.uploadStream() as writable: yield writable info.save() - log.debug("Wrote %r from stream.", info) + logger.debug("Wrote %r from stream.", info) def fileExists(self, jobStoreFileID): return self.FileInfo.exists(jobStoreFileID) @@ -574,13 +580,13 @@ def getFileSize(self, jobStoreFileID): def readFile(self, jobStoreFileID, localFilePath, symlink=False): info = self.FileInfo.loadOrFail(jobStoreFileID) - log.debug("Reading %r into %r.", info, localFilePath) + logger.debug("Reading %r into %r.", info, localFilePath) info.download(localFilePath, not self.config.disableJobStoreChecksumVerification) @contextmanager def readFileStream(self, jobStoreFileID): info = self.FileInfo.loadOrFail(jobStoreFileID) - log.debug("Reading %r into stream.", info) + logger.debug("Reading %r into stream.", info) with info.downloadStream() as readable: yield readable @@ -589,14 +595,14 @@ def readSharedFileStream(self, sharedFileName): self._requireValidSharedFileName(sharedFileName) jobStoreFileID = self._sharedFileID(sharedFileName) info = self.FileInfo.loadOrFail(jobStoreFileID, customName=sharedFileName) - log.debug("Reading %r for shared file %r into stream.", info, sharedFileName) + logger.debug("Reading %r for shared file %r into stream.", info, sharedFileName) with info.downloadStream() as readable: yield readable def deleteFile(self, jobStoreFileID): info = self.FileInfo.load(jobStoreFileID) if info is None: - log.debug("File %s does not exist, skipping deletion.", jobStoreFileID) + logger.debug("File %s does not exist, skipping deletion.", jobStoreFileID) else: info.delete() @@ -703,7 +709,7 @@ def _bindBucket(self, bucket_name, create=False, block=True, versioning=False, """ assert self.minBucketNameLen <= len(bucket_name) <= self.maxBucketNameLen assert self.bucketNameRe.match(bucket_name) - log.debug("Binding to job store bucket '%s'.", bucket_name) + logger.debug("Binding to job store bucket '%s'.", bucket_name) def bucket_creation_pending(e): # https://github.com/BD2KGenomics/toil/issues/955 @@ -722,9 +728,9 @@ def bucket_creation_pending(e): except S3ResponseError as e: if e.error_code == 'NoSuchBucket': bucketExisted = False - log.debug("Bucket '%s' does not exist.", bucket_name) + logger.debug("Bucket '%s' does not exist.", bucket_name) if create: - log.debug("Creating bucket '%s'.", bucket_name) + logger.debug("Creating bucket '%s'.", bucket_name) location = region_to_bucket_location(self.region) bucket = self.s3.create_bucket(bucket_name, location=location) # It is possible for create_bucket to return but @@ -757,7 +763,7 @@ def bucket_creation_pending(e): # consistent? time.sleep(1) while self._getBucketVersioning(bucket) != True: - log.warning("Waiting for versioning activation on bucket '%s'...", bucket_name) + logger.warning("Waiting for versioning activation on bucket '%s'...", bucket_name) time.sleep(1) elif check_versioning_consistency: # now test for versioning consistency @@ -768,9 +774,9 @@ def bucket_creation_pending(e): elif bucket_versioning is None: assert False, 'Cannot use a bucket with versioning suspended' if bucketExisted: - log.debug("Using pre-existing job store bucket '%s'.", bucket_name) + logger.debug("Using pre-existing job store bucket '%s'.", bucket_name) else: - log.debug("Created new job store bucket '%s' with versioning state %s.", bucket_name, str(versioning)) + logger.debug("Created new job store bucket '%s' with versioning state %s.", bucket_name, str(versioning)) return bucket @@ -790,7 +796,7 @@ def _bindDomain(self, domain_name, create=False, block=True): :raises SDBResponseError: If `block` is True and the domain still doesn't exist after the retry timeout expires. """ - log.debug("Binding to job store domain '%s'.", domain_name) + logger.debug("Binding to job store domain '%s'.", domain_name) retryargs = dict(predicate=lambda e: no_such_sdb_domain(e) or sdb_unavailable(e)) if not block: retryargs['timeout'] = 15 @@ -1104,7 +1110,7 @@ def _start_checksum(self, to_match=None, algorithm='sha1'): wrapped = getattr(hashlib, algorithm)() - log.debug('Starting %s checksum to match %s', algorithm, expected) + logger.debug('Starting %s checksum to match %s', algorithm, expected) return (algorithm, wrapped, expected) @@ -1112,7 +1118,6 @@ def _update_checksum(self, checksum_in_progress, data): """ Update a checksum in progress from _start_checksum with new data. """ - log.debug('Updating checksum with %d bytes', len(data)) checksum_in_progress[1].update(data) def _finish_checksum(self, checksum_in_progress): @@ -1122,8 +1127,8 @@ def _finish_checksum(self, checksum_in_progress): """ result_hash = checksum_in_progress[1].hexdigest() - - log.debug('Completed checksum with hash %s vs. expected %s', result_hash, checksum_in_progress[2]) + + logger.debug('Completed checksum with hash %s vs. expected %s', result_hash, checksum_in_progress[2]) if checksum_in_progress[2] is not None: # We expected a particular hash @@ -1166,19 +1171,20 @@ def readFrom(self, readable): assert isinstance(buf, bytes) if allowInlining and len(buf) <= info.maxInlinedSize(): - log.debug('Inlining content of %d bytes', len(buf)) + logger.debug('Inlining content of %d bytes', len(buf)) info.content = buf # There will be no checksum info.checksum = '' else: # We will compute a checksum hasher = info._start_checksum() + logger.debug('Updating checksum with %d bytes', len(buf)) info._update_checksum(hasher, buf) headers = info._s3EncryptionHeaders() for attempt in retry_s3(): with attempt: - log.debug('Starting multipart upload') + logger.debug('Starting multipart upload') upload = store.filesBucket.initiate_multipart_upload( key_name=compat_bytes(info.fileID), headers=headers) @@ -1186,7 +1192,7 @@ def readFrom(self, readable): for part_num in itertools.count(): for attempt in retry_s3(): with attempt: - log.debug('Uploading part %d of %d bytes', part_num + 1, len(buf)) + logger.debug('Uploading part %d of %d bytes', part_num + 1, len(buf)) upload.upload_part_from_file(fp=BytesIO(buf), # part numbers are 1-based part_num=part_num + 1, @@ -1200,14 +1206,14 @@ def readFrom(self, readable): break info._update_checksum(hasher, buf) except: - with panic(log=log): + with panic(log=logger): for attempt in retry_s3(): with attempt: upload.cancel_upload() else: while store._getBucketVersioning(store.filesBucket) != True: - log.warning('Versioning does not appear to be enabled yet. Deferring multipart upload completion...') + logger.warning('Versioning does not appear to be enabled yet. Deferring multipart upload completion...') time.sleep(1) # Save the checksum @@ -1215,18 +1221,18 @@ def readFrom(self, readable): for attempt in retry_s3(): with attempt: - log.debug('Attempting to complete upload...') + logger.debug('Attempting to complete upload...') completed = upload.complete_upload() - log.debug('Completed upload object of type %s: %s', str(type(completed)), repr(completed)) + logger.debug('Completed upload object of type %s: %s', str(type(completed)), repr(completed)) info.version = completed.version_id - log.debug('Completed upload with version %s', str(info.version)) + logger.debug('Completed upload with version %s', str(info.version)) if info.version is None: # Somehow we don't know the version. Try and get it. for attempt in retry_s3(predicate=lambda e: retryable_s3_errors(e) or isinstance(e, AssertionError)): with attempt: key = store.filesBucket.get_key(compat_bytes(info.fileID), headers=headers) - log.warning('Loaded key for upload with no version and got version %s', str(key.version_id)) + logger.warning('Loaded key for upload with no version and got version %s', str(key.version_id)) info.version = key.version_id assert info.version is not None @@ -1239,7 +1245,7 @@ def readFrom(self, readable): assert isinstance(buf, bytes) dataLength = len(buf) if allowInlining and dataLength <= info.maxInlinedSize(): - log.debug('Inlining content of %d bytes', len(buf)) + logger.debug('Inlining content of %d bytes', len(buf)) info.content = buf # There will be no checksum info.checksum = '' @@ -1254,17 +1260,17 @@ def readFrom(self, readable): headers = info._s3EncryptionHeaders() while store._getBucketVersioning(store.filesBucket) != True: - log.warning('Versioning does not appear to be enabled yet. Deferring single part upload...') + logger.warning('Versioning does not appear to be enabled yet. Deferring single part upload...') time.sleep(1) for attempt in retry_s3(): with attempt: - log.debug('Uploading single part of %d bytes', dataLength) + logger.debug('Uploading single part of %d bytes', dataLength) assert dataLength == key.set_contents_from_file(fp=buf, headers=headers) - log.debug('Upload received version %s', str(key.version_id)) + logger.debug('Upload received version %s', str(key.version_id)) info.version = key.version_id if info.version is None: @@ -1272,7 +1278,7 @@ def readFrom(self, readable): for attempt in retry_s3(predicate=lambda e: retryable_s3_errors(e) or isinstance(e, AssertionError)): with attempt: key.reload() - log.warning('Reloaded key with no version and got version %s', str(key.version_id)) + logger.warning('Reloaded key with no version and got version %s', str(key.version_id)) info.version = key.version_id assert info.version is not None @@ -1284,7 +1290,7 @@ def readFrom(self, readable): yield writable if not pipe.reader_done: - log.debug('Version: {} Content: {}'.format(self.version, self.content)) + logger.debug('Version: {} Content: {}'.format(self.version, self.content)) raise RuntimeError('Escaped context manager without written data being read!') # We check our work to make sure we have exactly one of embedded @@ -1292,11 +1298,11 @@ def readFrom(self, readable): if self.content is None: if not bool(self.version): - log.debug('Version: {} Content: {}'.format(self.version, self.content)) + logger.debug('Version: {} Content: {}'.format(self.version, self.content)) raise RuntimeError('No content added and no version created') else: if bool(self.version): - log.debug('Version: {} Content: {}'.format(self.version, self.content)) + logger.debug('Version: {} Content: {}'.format(self.version, self.content)) raise RuntimeError('Content added and version created') def copyFrom(self, srcKey): @@ -1309,11 +1315,11 @@ def copyFrom(self, srcKey): if srcKey.size <= self.maxInlinedSize(): self.content = srcKey.get_contents_as_string() else: - self.version = copyKeyMultipart(srcBucketName=compat_plain(srcKey.bucket.name), - srcKeyName=compat_plain(srcKey.name), - srcKeyVersion=compat_plain(srcKey.version_id), - dstBucketName=compat_plain(self.outer.filesBucket.name), - dstKeyName=compat_plain(self._fileID), + self.version = copyKeyMultipart(srcBucketName=compat_bytes(srcKey.bucket.name), + srcKeyName=compat_bytes(srcKey.name), + srcKeyVersion=compat_bytes(srcKey.version_id), + dstBucketName=compat_bytes(self.outer.filesBucket.name), + dstKeyName=compat_bytes(self._fileID), sseAlgorithm='AES256', sseKey=self._getSSEKey()) @@ -1336,11 +1342,11 @@ def copyTo(self, dstKey): srcKey = self.outer.filesBucket.get_key(compat_bytes(self.fileID)) srcKey.version_id = self.version with attempt: - copyKeyMultipart(srcBucketName=compat_plain(srcKey.bucket.name), - srcKeyName=compat_plain(srcKey.name), - srcKeyVersion=compat_plain(srcKey.version_id), - dstBucketName=compat_plain(dstKey.bucket.name), - dstKeyName=compat_plain(dstKey.name), + copyKeyMultipart(srcBucketName=compat_bytes(srcKey.bucket.name), + srcKeyName=compat_bytes(srcKey.name), + srcKeyVersion=compat_bytes(srcKey.version_id), + dstBucketName=compat_bytes(dstKey.bucket.name), + dstKeyName=compat_bytes(dstKey.name), copySourceSseAlgorithm='AES256', copySourceSseKey=self._getSSEKey()) else: @@ -1542,9 +1548,7 @@ def _delete_domain(self, domain): try: domain.delete() except SDBResponseError as e: - if no_such_sdb_domain(e): - pass - else: + if not no_such_sdb_domain(e): raise def _delete_bucket(self, b): diff --git a/src/toil/jobStores/aws/utils.py b/src/toil/jobStores/aws/utils.py index 0c2fc18a8d..beda1e6713 100644 --- a/src/toil/jobStores/aws/utils.py +++ b/src/toil/jobStores/aws/utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import base64 -import boto3 import bz2 import errno import itertools @@ -22,16 +21,18 @@ import types from ssl import SSLError -from toil.lib.compatibility import compat_bytes, compat_oldstr -from toil.lib.exceptions import panic -from toil.lib.retry import old_retry -from boto.exception import (SDBResponseError, - BotoServerError, - S3ResponseError, +import boto3 +from boto.exception import (BotoServerError, + S3CopyError, S3CreateError, - S3CopyError) + S3ResponseError, + SDBResponseError) from botocore.exceptions import ClientError +from toil.lib.compatibility import compat_bytes, compat_bytes +from toil.lib.exceptions import panic +from toil.lib.retry import old_retry + log = logging.getLogger(__name__) @@ -264,11 +265,11 @@ def copyKeyMultipart(srcBucketName, srcKeyName, srcKeyVersion, dstBucketName, ds :return: The version of the copied file (or None if versioning is not enabled for dstBucket). """ s3 = boto3.resource('s3') - dstBucket = s3.Bucket(compat_oldstr(dstBucketName)) - dstObject = dstBucket.Object(compat_oldstr(dstKeyName)) - copySource = {'Bucket': compat_oldstr(srcBucketName), 'Key': compat_oldstr(srcKeyName)} + dstBucket = s3.Bucket(compat_bytes(dstBucketName)) + dstObject = dstBucket.Object(compat_bytes(dstKeyName)) + copySource = {'Bucket': compat_bytes(srcBucketName), 'Key': compat_bytes(srcKeyName)} if srcKeyVersion is not None: - copySource['VersionId'] = compat_oldstr(srcKeyVersion) + copySource['VersionId'] = compat_bytes(srcKeyVersion) # The boto3 functions don't allow passing parameters as None to # indicate they weren't provided. So we have to do a bit of work diff --git a/src/toil/jobStores/conftest.py b/src/toil/jobStores/conftest.py index 14e362ec7d..bc402b4dec 100644 --- a/src/toil/jobStores/conftest.py +++ b/src/toil/jobStores/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/jobStores/fileJobStore.py b/src/toil/jobStores/fileJobStore.py index 7f43ca6b6d..6de3e4cf52 100644 --- a/src/toil/jobStores/fileJobStore.py +++ b/src/toil/jobStores/fileJobStore.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,7 +31,9 @@ NoSuchFileException, NoSuchJobException, NoSuchJobStoreException) -from toil.lib.misc import (AtomicFileCreate, atomic_copy, atomic_copyobj, +from toil.lib.misc import (AtomicFileCreate, + atomic_copy, + atomic_copyobj, robust_rmtree) logger = logging.getLogger(__name__) diff --git a/src/toil/jobStores/googleJobStore.py b/src/toil/jobStores/googleJobStore.py index d8cb9a177b..f46aa6f032 100644 --- a/src/toil/jobStores/googleJobStore.py +++ b/src/toil/jobStores/googleJobStore.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/jobStores/utils.py b/src/toil/jobStores/utils.py index 54caca0f59..85e480dcdf 100644 --- a/src/toil/jobStores/utils.py +++ b/src/toil/jobStores/utils.py @@ -287,6 +287,3 @@ def transform(self, readable, writable): def writeTo(self, writable): self.transform(self.source, writable) - - - diff --git a/src/toil/leader.py b/src/toil/leader.py index 73ec5cb4f1..e249bc949e 100644 --- a/src/toil/leader.py +++ b/src/toil/leader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,27 +24,28 @@ import sys import time -from toil import resolveEntryPoint -from toil.lib.humanize import bytes2human - -try: - from toil.cwl.cwltoil import CWL_INTERNAL_JOBS -except ImportError: - # CWL extra not installed - CWL_INTERNAL_JOBS = () import enlighten +from toil import resolveEntryPoint from toil.batchSystems import DeadlockException from toil.batchSystems.abstractBatchSystem import BatchJobExitReason from toil.common import Toil, ToilMetrics from toil.job import CheckpointJobDescription, ServiceJobDescription from toil.jobStores.abstractJobStore import NoSuchJobException +from toil.lib.humanize import bytes2human from toil.lib.throttle import LocalThrottle from toil.provisioners.clusterScaler import ScalerThread from toil.serviceManager import ServiceManager from toil.statsAndLogging import StatsAndLogging from toil.toilState import ToilState +try: + from toil.cwl.cwltoil import CWL_INTERNAL_JOBS +except ImportError: + # CWL extra not installed + CWL_INTERNAL_JOBS = () + + logger = logging.getLogger( __name__ ) ############################################################################### @@ -63,11 +64,6 @@ ############################################################################### - -#################################################### -# Exception thrown by the Leader class when one or more jobs fails -#################################################### - class FailedJobsException(Exception): def __init__(self, jobStoreLocator, failedJobs, jobStore): self.msg = "The job store '%s' contains %i failed jobs" % (jobStoreLocator, len(failedJobs)) @@ -86,13 +82,13 @@ def __init__(self, jobStoreLocator, failedJobs, jobStore): super().__init__() self.jobStoreLocator = jobStoreLocator self.numberOfFailedJobs = len(failedJobs) - + def __str__(self): """ Stringify the exception, including the message. """ return self.msg - + #################################################### ##Following class represents the leader diff --git a/src/toil/lib/bioio.py b/src/toil/lib/bioio.py index 7a13e1c749..b745246978 100644 --- a/src/toil/lib/bioio.py +++ b/src/toil/lib/bioio.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,292 +12,46 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import logging.handlers -import os -import random -import resource import subprocess -import tempfile -from argparse import ArgumentParser -defaultLogLevel = logging.INFO -logger = logging.getLogger(__name__) -rootLogger = logging.getLogger() -toilLogger = logging.getLogger('toil') - - -def getLogLevelString(logger=None): - if logger is None: - logger = rootLogger - return logging.getLevelName(logger.getEffectiveLevel()) - -__loggingFiles = [] -def addLoggingFileHandler(fileName, rotatingLogging=False): - if fileName in __loggingFiles: - return - __loggingFiles.append(fileName) - if rotatingLogging: - handler = logging.handlers.RotatingFileHandler(fileName, maxBytes=1000000, backupCount=1) - else: - handler = logging.FileHandler(fileName) - rootLogger.addHandler(handler) - return handler - - -def setLogLevel(level, logger=None): - """ - Sets the log level to a given string level (like "INFO"). Operates on the - root logger by default, but another logger can be specified instead. - """ - if logger is None: - logger = rootLogger - level = level.upper() - if level == "OFF": level = "CRITICAL" - # Note that getLevelName works in both directions, numeric to textual and textual to numeric - numericLevel = logging.getLevelName(level) - assert logging.getLevelName(numericLevel) == level - logger.setLevel(numericLevel) - # There are quite a few cases where we expect AWS requests to fail, but it seems - # that boto handles these by logging the error *and* raising an exception. We - # don't want to confuse the user with those error messages. - logging.getLogger( 'boto' ).setLevel( logging.CRITICAL ) - -def logFile(fileName, printFunction=logger.info): - """Writes out a formatted version of the given log file - """ - printFunction("Reporting file: %s" % fileName) - shortName = fileName.split("/")[-1] - fileHandle = open(fileName, 'r') - line = fileHandle.readline() - while line != '': - if line[-1] == '\n': - line = line[:-1] - printFunction("%s:\t%s" % (shortName, line)) - line = fileHandle.readline() - fileHandle.close() - -def logStream(fileHandle, shortName, printFunction=logger.info): - """Writes out a formatted version of the given log stream. - """ - printFunction("Reporting file: %s" % shortName) - line = fileHandle.readline() - while line != '': - if line[-1] == '\n': - line = line[:-1] - printFunction("%s:\t%s" % (shortName, line)) - line = fileHandle.readline() - fileHandle.close() - -def addLoggingOptions(parser): - # Wrapper function that allows toil to be used with both the optparse and - # argparse option parsing modules - if isinstance(parser, ArgumentParser): - group = parser.add_argument_group("Logging Options", - "Options that control logging") - _addLoggingOptions(group.add_argument) - else: - raise RuntimeError("Unanticipated class passed to " - "addLoggingOptions(), %s. Expecting " - "argparse.ArgumentParser" % parser.__class__) - -supportedLogLevels = (logging.CRITICAL, logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG) - -def _addLoggingOptions(addOptionFn): - """ - Adds logging options - """ - # BEFORE YOU ADD OR REMOVE OPTIONS TO THIS FUNCTION, KNOW THAT YOU MAY ONLY USE VARIABLES ACCEPTED BY BOTH - # optparse AND argparse FOR EXAMPLE, YOU MAY NOT USE default=%default OR default=%(default)s - defaultLogLevelName = logging.getLevelName( defaultLogLevel ) - addOptionFn("--logOff", dest="logLevel", - default=defaultLogLevelName, - action="store_const", const="CRITICAL", - help="Same as --logCritical") - for level in supportedLogLevels: - levelName = logging.getLevelName(level) - levelNameCapitalized = levelName.capitalize() - addOptionFn("--log" + levelNameCapitalized, dest="logLevel", - default=defaultLogLevelName, - action="store_const", const=levelName, - help="Turn on logging at level %s and above. (default is %s)" % (levelName, defaultLogLevelName)) - addOptionFn("--logLevel", dest="logLevel", default=defaultLogLevelName, - help=("Log at given level (may be either OFF (or CRITICAL), ERROR, WARN (or WARNING), INFO or DEBUG). " - "(default is %s)" % defaultLogLevelName)) - addOptionFn("--logFile", dest="logFile", help="File to log in") - addOptionFn("--rotatingLogging", dest="logRotating", action="store_true", default=False, - help="Turn on rotating logging, which prevents log files getting too big.") - -def configureRootLogger(): - """ - Set up the root logger with handlers and formatting. - - Should be called (either by itself or via setLoggingFromOptions) before any - entry point tries to log anything, to ensure consistent formatting. - """ - - formatStr = ' '.join(['[%(asctime)s]', '[%(threadName)-10s]', - '[%(levelname).1s]', '[%(name)s]', '%(message)s']) - logging.basicConfig(format=formatStr, datefmt='%Y-%m-%dT%H:%M:%S%z') - rootLogger.setLevel(defaultLogLevel) - -def setLoggingFromOptions(options): - """ - Sets the logging from a dictionary of name/value options. - """ - configureRootLogger() - if options.logLevel is not None: - setLogLevel(options.logLevel) - else: - # Ensure that any other log level overrides are in effect even if no log level is explicitly set - setLogLevel(getLogLevelString()) - logger.debug("Root logger is at level '%s', 'toil' logger at level '%s'.", - getLogLevelString(logger=rootLogger), getLogLevelString(logger=toilLogger)) - if options.logFile is not None: - addLoggingFileHandler(options.logFile, rotatingLogging=options.logRotating) - logger.debug("Logging to file '%s'." % options.logFile) +from toil.statsAndLogging import (logger, + root_logger, + set_logging_from_options) +from toil.test import get_temp_file +# used by cactus +# TODO: only used in utilsTest.py; move this there once out of cactus def system(command): """ A convenience wrapper around subprocess.check_call that logs the command before passing it on. The command can be either a string or a sequence of strings. If it is a string shell=True will be passed to subprocess.check_call. - :type command: str | sequence[string] """ - logger.debug('Running: %r', command) + logger.warning('Deprecated toil method that will be moved/replaced in a future release."') + logger.debug(f'Running: {command}') subprocess.check_call(command, shell=isinstance(command, str), bufsize=-1) -def getTotalCpuTimeAndMemoryUsage(): - """ - Gives the total cpu time of itself and all its children, and the maximum RSS memory usage of - itself and its single largest child. - """ - me = resource.getrusage(resource.RUSAGE_SELF) - childs = resource.getrusage(resource.RUSAGE_CHILDREN) - totalCPUTime = me.ru_utime + me.ru_stime + childs.ru_utime + childs.ru_stime - totalMemoryUsage = me.ru_maxrss + childs.ru_maxrss - return totalCPUTime, totalMemoryUsage -def getTotalCpuTime(): - """Gives the total cpu time, including the children. - """ - return getTotalCpuTimeAndMemoryUsage()[0] - -def getTotalMemoryUsage(): - """Gets the amount of memory used by the process and its largest child. - """ - return getTotalCpuTimeAndMemoryUsage()[1] - -######################################################### -######################################################### -######################################################### -#testing settings -######################################################### -######################################################### -######################################################### - -class TestStatus(object): - ###Global variables used by testing framework to run tests. - TEST_SHORT = 0 - TEST_MEDIUM = 1 - TEST_LONG = 2 - TEST_VERY_LONG = 3 - - TEST_STATUS = TEST_SHORT - - SAVE_ERROR_LOCATION = None - - def getTestStatus(): - return TestStatus.TEST_STATUS - getTestStatus = staticmethod(getTestStatus) - - def setTestStatus(status): - assert status in (TestStatus.TEST_SHORT, TestStatus.TEST_MEDIUM, TestStatus.TEST_LONG, TestStatus.TEST_VERY_LONG) - TestStatus.TEST_STATUS = status - setTestStatus = staticmethod(setTestStatus) - - def getSaveErrorLocation(): - """Location to in which to write inputs which created test error. - """ - return TestStatus.SAVE_ERROR_LOCATION - getSaveErrorLocation = staticmethod(getSaveErrorLocation) - - def setSaveErrorLocation(dir): - """Set location in which to write inputs which created test error. - """ - logger.debug("Location to save error files in: %s" % dir) - assert os.path.isdir(dir) - TestStatus.SAVE_ERROR_LOCATION = dir - setSaveErrorLocation = staticmethod(setSaveErrorLocation) - - def getTestSetup(shortTestNo=1, mediumTestNo=5, longTestNo=100, veryLongTestNo=0): - if TestStatus.TEST_STATUS == TestStatus.TEST_SHORT: - return shortTestNo - elif TestStatus.TEST_STATUS == TestStatus.TEST_MEDIUM: - return mediumTestNo - elif TestStatus.TEST_STATUS == TestStatus.TEST_LONG: - return longTestNo - else: #Used for long example tests - return veryLongTestNo - getTestSetup = staticmethod(getTestSetup) - - def getPathToDataSets(): - """This method is used to store the location of - the path where all the data sets used by tests for analysis are kept. - These are not kept in the distrbution itself for reasons of size. - """ - assert "SON_TRACE_DATASETS" in os.environ - return os.environ["SON_TRACE_DATASETS"] - getPathToDataSets = staticmethod(getPathToDataSets) - -def getBasicOptionParser( parser=None): - if parser is None: - parser = ArgumentParser() - - addLoggingOptions(parser) - - parser.add_argument("--tempDirRoot", dest="tempDirRoot", type=str, - help="Path to where temporary directory containing all temp files are created, by default uses the current working directory as the base.", - default=tempfile.gettempdir()) - - return parser - -def parseBasicOptions(parser): - """Setups the standard things from things added by getBasicOptionParser. - """ - options = parser.parse_args() - - setLoggingFromOptions(options) - - #Set up the temp dir root - if options.tempDirRoot == "None": # FIXME: Really, a string containing the word None? - options.tempDirRoot = tempfile.gettempdir() +# Used by cactus; now a wrapper and not used in Toil. +# TODO: Remove from cactus and then remove from Toil. +def getLogLevelString(logger=None): + root_logger.warning('Deprecated toil method. Please call "logging.getLevelName" directly.') + if logger is None: + logger = root_logger + return logging.getLevelName(logger.getEffectiveLevel()) - return options -def getRandomAlphaNumericString(length=10): - """Returns a random alpha numeric string of the given length. - """ - return "".join([ random.choice('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz') for i in range(0, length) ]) +# Used by cactus; now a wrapper and not used in Toil. +# TODO: Remove from cactus and then remove from Toil. +def setLoggingFromOptions(options): + logger.warning('Deprecated toil method. Please use "toil.statsAndLogging.set_logging_from_options()" instead."') + set_logging_from_options(options) -def makePublicDir(dirName): - """Makes a given subdirectory if it doesn't already exist, making sure it is public. - """ - if not os.path.exists(dirName): - os.mkdir(dirName) - os.chmod(dirName, 0o777) - return dirName +# Used by cactus; now a wrapper and not used in Toil. +# TODO: Remove from cactus and then remove from Toil. def getTempFile(suffix="", rootDir=None): - """Returns a string representing a temporary file, that must be manually deleted - """ - if rootDir is None: - handle, tmpFile = tempfile.mkstemp(suffix) - os.close(handle) - return tmpFile - else: - tmpFile = os.path.join(rootDir, "tmp_" + getRandomAlphaNumericString() + suffix) - open(tmpFile, 'w').close() - os.chmod(tmpFile, 0o777) #Ensure everyone has access to the file. - return tmpFile + logger.warning('Deprecated toil method. Please use "toil.test.get_temp_file()" instead."') + return get_temp_file(suffix, rootDir) diff --git a/src/toil/lib/compatibility.py b/src/toil/lib/compatibility.py index 95996bb069..3f32d2fa44 100644 --- a/src/toil/lib/compatibility.py +++ b/src/toil/lib/compatibility.py @@ -1,10 +1,2 @@ -def compat_oldstr(s): - return s.decode('utf-8') if isinstance(s, bytes) else s - - def compat_bytes(s): return s.decode('utf-8') if isinstance(s, bytes) else s - - -def compat_plain(s): - return s.decode('utf-8') if isinstance(s, bytes) else s diff --git a/src/toil/lib/context.py b/src/toil/lib/context.py index b5759036c7..78a1d8b5ea 100644 --- a/src/toil/lib/context.py +++ b/src/toil/lib/context.py @@ -1,9 +1,22 @@ +# Copyright (C) 2015-2021 Regents of the University of California +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import json import logging import os import re - from urllib.parse import unquote + from boto import iam, sns, sqs, vpc from boto.exception import BotoServerError from boto.s3.connection import S3Connection @@ -12,7 +25,7 @@ from toil.lib.ec2 import UserError from toil.lib.memoize import memoize -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) class Context(object): @@ -468,7 +481,7 @@ def iam_user_name(self): try: return self.iam.get_user().user_name except BaseException: - log.warning("IAMConnection.get_user() failed.", exc_info=True) + logger.warning("IAMConnection.get_user() failed.", exc_info=True) return None current_user_placeholder = '__me__' diff --git a/src/toil/lib/docker.py b/src/toil/lib/docker.py index b96ff17c3d..2f5f565f4c 100644 --- a/src/toil/lib/docker.py +++ b/src/toil/lib/docker.py @@ -1,5 +1,16 @@ - - +# Copyright (C) 2015-2021 Regents of the University of California +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import base64 import logging import os @@ -10,7 +21,9 @@ import requests import docker -from docker.errors import (ContainerError, ImageNotFound, NotFound, +from docker.errors import (ContainerError, + ImageNotFound, + NotFound, create_api_error_from_http_exception) from docker.utils.socket import consume_socket_output, demux_adaptor diff --git a/src/toil/lib/ec2.py b/src/toil/lib/ec2.py index f1be921883..15da235dd9 100644 --- a/src/toil/lib/ec2.py +++ b/src/toil/lib/ec2.py @@ -1,3 +1,4 @@ +import boto3 import logging import time from operator import attrgetter @@ -7,12 +8,14 @@ from boto.exception import EC2ResponseError from toil.lib.exceptions import panic -from toil.lib.retry import old_retry +from toil.lib.retry import old_retry, retry a_short_time = 5 a_long_time = 60 * 60 log = logging.getLogger(__name__) +iam_client = boto3.client('iam') + class UserError(RuntimeError): def __init__(self, message=None, cause=None): @@ -255,20 +258,28 @@ def create_ondemand_instances(ec2, image_id, spec, num_instances=1): **spec).instances -# TODO: Implement retry_decorator here -# [5, 5, 10, 20, 20, 20, 20] I don't think we need to retry for an hour... ??? -# InvalidGroup.NotFound -# OR -# 'invalid iam instance profile' in m.lower() or 'no associated iam roles' in m.lower() +# exception is generated by a factory so we weirdly need a client instance to reference it +@retry(errors=[iam_client.exceptions.NoSuchEntityException]) +def wait_until_instance_profile_arn_exists(instance_profile_arn: Dict): + # The Arn and Name keys are mutually exclusive; we expect 'Arn' here. + if 'Arn' in instance_profile_arn: + instance_profile_name = instance_profile_arn['Arn'].split(':instance-profile/')[-1] + iam_client.get_instance_profile(InstanceProfileName=instance_profile_name) + elif 'Name' in instance_profile_arn: + instance_profile_name = instance_profile_arn['Name'] + iam_client.get_instance_profile(InstanceProfileName=instance_profile_name) + + +@retry(intervals=[5, 5, 10, 20, 20, 20, 20]) def create_instances(ec2: ServiceResource, image_id: str, key_name: str, instance_type: str, + instance_profile_arn: Dict, num_instances: int = 1, security_group_ids: Optional[List] = None, user_data: Optional[bytes] = None, block_device_map: Optional[List[Dict]] = None, - instance_profile_arn: Optional[Dict] = None, placement: Optional[Dict] = None, subnet_id: str = None): """ @@ -280,24 +291,23 @@ def create_instances(ec2: ServiceResource, https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Client.run_instances """ log.info('Creating %s instance(s) ... ', instance_type) - for attempt in retry_ec2(retry_for=a_long_time, retry_while=inconsistencies_detected): - with attempt: - request = {'ImageId': image_id, - 'MinCount': num_instances, - 'MaxCount': num_instances, - 'KeyName': key_name, - 'SecurityGroupIds': security_group_ids, - 'InstanceType': instance_type, - 'UserData': user_data, - 'Placement': placement, - 'BlockDeviceMappings': block_device_map, - 'IamInstanceProfile': instance_profile_arn, - 'SubnetId': subnet_id} - - # remove empty args - actual_request = dict() - for key in request: - if request[key]: - actual_request[key] = request[key] - - return ec2.create_instances(**actual_request) + wait_until_instance_profile_arn_exists(instance_profile_arn) + request = {'ImageId': image_id, + 'MinCount': num_instances, + 'MaxCount': num_instances, + 'KeyName': key_name, + 'SecurityGroupIds': security_group_ids, + 'InstanceType': instance_type, + 'UserData': user_data, + 'Placement': placement, + 'BlockDeviceMappings': block_device_map, + 'IamInstanceProfile': instance_profile_arn, + 'SubnetId': subnet_id} + + # remove empty args + actual_request = dict() + for key in request: + if request[key]: + actual_request[key] = request[key] + + return ec2.create_instances(**actual_request) diff --git a/src/toil/lib/ec2nodes.py b/src/toil/lib/ec2nodes.py index 98457496bd..706b62e1c7 100644 --- a/src/toil/lib/ec2nodes.py +++ b/src/toil/lib/ec2nodes.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/lib/encryption/__init__.py b/src/toil/lib/encryption/__init__.py index 7bdaff7be3..78a02ffd52 100644 --- a/src/toil/lib/encryption/__init__.py +++ b/src/toil/lib/encryption/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/lib/encryption/_dummy.py b/src/toil/lib/encryption/_dummy.py index 1b270b30b2..a77206b423 100644 --- a/src/toil/lib/encryption/_dummy.py +++ b/src/toil/lib/encryption/_dummy.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/lib/encryption/_nacl.py b/src/toil/lib/encryption/_nacl.py index b8ef052c69..1e90553111 100644 --- a/src/toil/lib/encryption/_nacl.py +++ b/src/toil/lib/encryption/_nacl.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/lib/exceptions.py b/src/toil/lib/exceptions.py index 7719188eff..9f6f1e4c93 100644 --- a/src/toil/lib/exceptions.py +++ b/src/toil/lib/exceptions.py @@ -59,4 +59,3 @@ def raise_(exc_type, exc_value, traceback) -> None: if exc.__traceback__ is not traceback: raise exc.with_traceback(traceback) raise exc - diff --git a/src/toil/lib/expando.py b/src/toil/lib/expando.py index f6c972c2eb..d731b81a7e 100644 --- a/src/toil/lib/expando.py +++ b/src/toil/lib/expando.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -129,4 +129,3 @@ def __getattribute__( self, name ): child = self.__class__( ) self[name] = child return child - diff --git a/src/toil/lib/generatedEC2Lists.py b/src/toil/lib/generatedEC2Lists.py index dc18424f24..2e801c57d3 100644 --- a/src/toil/lib/generatedEC2Lists.py +++ b/src/toil/lib/generatedEC2Lists.py @@ -1,7 +1,7 @@ # !!! AUTOGENERATED FILE !!! # Update with: src/toil/utils/toilUpdateEC2Instances.py # -# Copyright (C) 2015-2020 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 UCSC Computational Genomics Lab # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/lib/iterables.py b/src/toil/lib/iterables.py index 9c15f87d69..bd2a1266de 100644 --- a/src/toil/lib/iterables.py +++ b/src/toil/lib/iterables.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/lib/memoize.py b/src/toil/lib/memoize.py index 9742ef93cc..29ae4795c5 100644 --- a/src/toil/lib/memoize.py +++ b/src/toil/lib/memoize.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/lib/objects.py b/src/toil/lib/objects.py index 28c43d44b3..d6c34d4462 100644 --- a/src/toil/lib/objects.py +++ b/src/toil/lib/objects.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/lib/resources.py b/src/toil/lib/resources.py new file mode 100644 index 0000000000..73136afdf5 --- /dev/null +++ b/src/toil/lib/resources.py @@ -0,0 +1,33 @@ +# Copyright (C) 2015-2021 Regents of the University of California +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import resource + + +def get_total_cpu_time_and_memory_usage(): + """ + Gives the total cpu time of itself and all its children, and the maximum RSS memory usage of + itself and its single largest child. + """ + me = resource.getrusage(resource.RUSAGE_SELF) + children = resource.getrusage(resource.RUSAGE_CHILDREN) + total_cpu_time = me.ru_utime + me.ru_stime + children.ru_utime + children.ru_stime + total_memory_usage = me.ru_maxrss + children.ru_maxrss + return total_cpu_time, total_memory_usage + + +def get_total_cpu_time(): + """Gives the total cpu time, including the children.""" + me = resource.getrusage(resource.RUSAGE_SELF) + childs = resource.getrusage(resource.RUSAGE_CHILDREN) + return me.ru_utime + me.ru_stime + childs.ru_utime + childs.ru_stime diff --git a/src/toil/lib/retry.py b/src/toil/lib/retry.py index e3a75f353a..202ce89ab4 100644 --- a/src/toil/lib/retry.py +++ b/src/toil/lib/retry.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -150,7 +150,7 @@ def boto_bucket(bucket_name): except ModuleNotFoundError: botocore = None -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) class ErrorCondition: @@ -257,7 +257,7 @@ def call(*args, **kwargs): raise interval = intervals_remaining.pop(0) - log.debug(f"Error in {func}: {e}. Retrying after {interval} s...") + logger.debug(f"Error in {func}: {e}. Retrying after {interval} s...") time.sleep(interval) return call return decorate @@ -424,7 +424,7 @@ def repeated_attempt( delay ): yield except Exception as e: if time.time( ) + delay < expiration and predicate( e ): - log.info( 'Got %s, trying again in %is.', e, delay ) + logger.info('Got %s, trying again in %is.', e, delay) time.sleep( delay ) else: raise diff --git a/src/toil/lib/threading.py b/src/toil/lib/threading.py index 5bd272beb7..4e74a944be 100644 --- a/src/toil/lib/threading.py +++ b/src/toil/lib/threading.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ from toil.lib.exceptions import raise_ from toil.lib.misc import robust_rmtree -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) class ExceptionalThread(threading.Thread): @@ -107,14 +107,14 @@ def cpu_count(): # Get the fallback answer of all the CPUs on the machine total_machine_size = psutil.cpu_count(logical=True) - log.debug('Total machine size: %d cores', total_machine_size) + logger.debug('Total machine size: %d cores', total_machine_size) try: with open('/sys/fs/cgroup/cpu/cpu.cfs_quota_us', 'r') as stream: # Read the quota quota = int(stream.read()) - log.debug('CPU quota: %d', quota) + logger.debug('CPU quota: %d', quota) if quota == -1: # Assume we can use the whole machine @@ -124,21 +124,21 @@ def cpu_count(): # Read the period in which we are allowed to burn the quota period = int(stream.read()) - log.debug('CPU quota period: %d', period) + logger.debug('CPU quota period: %d', period) # The thread count is how many multiples of a wall clcok period we can burn in that period. cgroup_size = int(math.ceil(float(quota)/float(period))) - log.debug('Cgroup size in cores: %d', cgroup_size) + logger.debug('Cgroup size in cores: %d', cgroup_size) except: # We can't actually read these cgroup fields. Maybe we are a mac or something. - log.debug('Could not inspect cgroup: %s', traceback.format_exc()) + logger.debug('Could not inspect cgroup: %s', traceback.format_exc()) cgroup_size = float('inf') # Return the smaller of the actual thread count and the cgroup's limit, minimum 1. result = max(1, min(cgroup_size, total_machine_size)) - log.debug('cpu_count: %s', str(result)) + logger.debug('cpu_count: %s', str(result)) # Make sure to remember it for the next call setattr(cpu_count, 'result', result) return result @@ -320,7 +320,7 @@ def global_mutex(workDir, mutex): # Define a filename lock_filename = os.path.join(workDir, 'toil-mutex-' + mutex) - log.debug('PID %d acquiring mutex %s', os.getpid(), lock_filename) + logger.debug('PID %d acquiring mutex %s', os.getpid(), lock_filename) # We can't just create/open and lock a file, because when we clean up # there's a race where someone can open the file before we unlink it and @@ -356,12 +356,12 @@ def global_mutex(workDir, mutex): try: # When we have it, do the thing we are protecting. - log.debug('PID %d now holds mutex %s', os.getpid(), lock_filename) + logger.debug('PID %d now holds mutex %s', os.getpid(), lock_filename) yield finally: # Delete it while we still own it, so we can't delete it from out from # under someone else who thinks they are holding it. - log.debug('PID %d releasing mutex %s', os.getpid(), lock_filename) + logger.debug('PID %d releasing mutex %s', os.getpid(), lock_filename) os.unlink(lock_filename) fcntl.lockf(fd, fcntl.LOCK_UN) # Note that we are unlinking it and then unlocking it; a lot of people @@ -426,7 +426,7 @@ def enter(self): You may not enter the arena again before leaving it. """ - log.debug('Joining arena %s', self.lockfileDir) + logger.debug('Joining arena %s', self.lockfileDir) # Make sure we're not in it already. assert self.lockfileName is None @@ -448,7 +448,7 @@ def enter(self): # Now we're properly in, so release the global mutex - log.debug('Now in arena %s', self.lockfileDir) + logger.debug('Now in arena %s', self.lockfileDir) def leave(self): """ @@ -468,7 +468,7 @@ def leave(self): assert self.lockfileName is not None assert self.lockfileFD is not None - log.debug('Leaving arena %s', self.lockfileDir) + logger.debug('Leaving arena %s', self.lockfileDir) with global_mutex(self.workDir, self.mutex): # Now nobody else should also be trying to join or leave. @@ -504,22 +504,17 @@ def leave(self): else: # Nothing alive was found. Nobody will come in while we hold # the global mutex, so we are the Last Process Standing. - log.debug('We are the Last Process Standing in arena %s', self.lockfileDir) + logger.debug('We are the Last Process Standing in arena %s', self.lockfileDir) yield True try: # Delete the arena directory so as to leave nothing behind. os.rmdir(self.lockfileDir) except: - log.warning('Could not clean up arena %s completely: %s', - self.lockfileDir, traceback.format_exc()) + logger.warning('Could not clean up arena %s completely: %s', + self.lockfileDir, traceback.format_exc()) # Now we're done, whether we were the last one or not, and can # release the mutex. - log.debug('Now out of arena %s', self.lockfileDir) - - - - - + logger.debug('Now out of arena %s', self.lockfileDir) diff --git a/src/toil/lib/throttle.py b/src/toil/lib/throttle.py index 4898fc9872..306e15a0d9 100644 --- a/src/toil/lib/throttle.py +++ b/src/toil/lib/throttle.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/provisioners/.gitignore b/src/toil/provisioners/.gitignore deleted file mode 100644 index 70bcc5309c..0000000000 --- a/src/toil/provisioners/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Generated by Ansible -*.retry diff --git a/src/toil/provisioners/__init__.py b/src/toil/provisioners/__init__.py index 64e477d6c1..c13e1b8986 100644 --- a/src/toil/provisioners/__init__.py +++ b/src/toil/provisioners/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging +from difflib import get_close_matches logger = logging.getLogger(__name__) -def clusterFactory(provisioner, clusterName=None, zone=None, nodeStorage=50, nodeStorageOverrides=None, sseKey=None): +def cluster_factory(provisioner, clusterName=None, zone=None, nodeStorage=50, nodeStorageOverrides=None, sseKey=None): """ :param clusterName: The name of the cluster. :param provisioner: The cloud type of the cluster. @@ -41,7 +41,66 @@ def clusterFactory(provisioner, clusterName=None, zone=None, nodeStorage=50, nod else: raise RuntimeError("Invalid provisioner '%s'" % provisioner) + +def add_provisioner_options(parser): + group = parser.add_argument_group("Provisioner Options.") + group.add_argument('-p', "--provisioner", dest='provisioner', choices=['aws', 'gce'], required=False, + default="aws", help="The provisioner for cluster auto-scaling. " + "AWS and Google are currently supported.") + group.add_argument('-z', '--zone', dest='zone', required=False, default=None, + help="The availability zone of the master. This parameter can also be set via the 'TOIL_X_ZONE' " + "environment variable, where X is AWS or GCE, or by the ec2_region_name parameter " + "in your .boto file, or derived from the instance metadata if using this utility on an " + "existing EC2 instance.") + group.add_argument("clusterName", help="The name that the cluster will be identifiable by. " + "Must be lowercase and may not contain the '_' character.") + + +def check_valid_node_types(provisioner, node_types): + """ + Raises if an invalid nodeType is specified for aws or gce. + + :param str provisioner: 'aws' or 'gce' to specify which cloud provisioner used. + :param node_types: A list of node types. Example: ['t2.micro', 't2.medium'] + :return: Nothing. Raises if invalid nodeType. + """ + if not node_types: + return + if not isinstance(node_types, list): + node_types = [node_types] + if not isinstance(node_types[0], str): + return + # check if a valid node type for aws + from toil.lib.generatedEC2Lists import E2Instances, regionDict + if provisioner == 'aws': + from toil.provisioners.aws import get_current_aws_region + current_region = get_current_aws_region() or 'us-west-2' + # check if instance type exists in this region + for nodeType in node_types: + if nodeType and ':' in nodeType: + nodeType = nodeType.split(':')[0] + if nodeType not in regionDict[current_region]: + # They probably misspelled it and can't tell. + close = get_close_matches(nodeType, regionDict[current_region], 1) + if len(close) > 0: + helpText = ' Did you mean ' + close[0] + '?' + else: + helpText = '' + raise RuntimeError(f'Invalid nodeType ({nodeType}) specified for AWS in ' + f'region: {current_region}.{helpText}') + elif provisioner == 'gce': + for nodeType in node_types: + if nodeType and ':' in nodeType: + nodeType = nodeType.split(':')[0] + + if nodeType not in E2Instances: + raise RuntimeError(f"It looks like you've specified an AWS nodeType with the {provisioner} " + f"provisioner. Please specify a nodeType for {provisioner}.") + else: + raise RuntimeError(f"Invalid provisioner: {provisioner}") + + class NoSuchClusterException(Exception): """Indicates that the specified cluster does not exist.""" - def __init__(self, clusterName): - super(NoSuchClusterException, self).__init__("The cluster '%s' could not be found" % clusterName) + def __init__(self, cluster_name): + super(NoSuchClusterException, self).__init__(f"The cluster '{cluster_name}' could not be found") diff --git a/src/toil/provisioners/abstractProvisioner.py b/src/toil/provisioners/abstractProvisioner.py index 3092219304..0004daa2b6 100644 --- a/src/toil/provisioners/abstractProvisioner.py +++ b/src/toil/provisioners/abstractProvisioner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/provisioners/aws/__init__.py b/src/toil/provisioners/aws/__init__.py index 00430251b1..3541bf3a1a 100644 --- a/src/toil/provisioners/aws/__init__.py +++ b/src/toil/provisioners/aws/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,18 +15,17 @@ import logging import os from collections import namedtuple -from difflib import get_close_matches from operator import attrgetter -from statistics import stdev, mean -from urllib.request import urlopen +from statistics import mean, stdev from urllib.error import URLError +from urllib.request import urlopen logger = logging.getLogger(__name__) ZoneTuple = namedtuple('ZoneTuple', ['name', 'price_deviation']) -def runningOnEC2(): +def running_on_ec2(): def file_begins_with(path, prefix): with open(path) as f: return f.read(len(prefix)) == prefix @@ -43,44 +42,41 @@ def file_begins_with(path, prefix): return False -def zoneToRegion(zone): +def zone_to_region(zone): """Get a region (e.g. us-west-2) from a zone (e.g. us-west-1c).""" from toil.lib.context import Context return Context.availability_zone_re.match(zone).group(1) -def getSpotZone(spotBid, nodeType, ctx): - return _getCurrentAWSZone(spotBid, nodeType, ctx) - +def get_current_aws_region(): + aws_zone = get_current_aws_zone() + return zone_to_region(aws_zone) if aws_zone else None -def getCurrentAWSZone(): - return _getCurrentAWSZone() - -def _getCurrentAWSZone(spotBid=None, nodeType=None, ctx=None): - zone = None +def get_current_aws_zone(spotBid=None, nodeType=None, ctx=None): try: import boto from boto.utils import get_instance_metadata except ImportError: - pass - else: - zone = os.environ.get('TOIL_AWS_ZONE', None) - if not zone and runningOnEC2(): - try: - zone = get_instance_metadata()['placement']['availability-zone'] - except KeyError: - pass - if not zone and spotBid: - # if spot bid is present, all the other parameters must be as well - assert bool(spotBid) == bool(nodeType) == bool(ctx) - # if the zone is unset and we are using the spot market, optimize our - # choice based on the spot history - return optimize_spot_bid(ctx=ctx, instance_type=nodeType, spot_bid=float(spotBid)) - if not zone: - zone = boto.config.get('Boto', 'ec2_region_name') - if zone is not None: - zone += 'a' # derive an availability zone in the region + return None + + zone = os.environ.get('TOIL_AWS_ZONE', None) + if not zone and running_on_ec2(): + try: + zone = get_instance_metadata()['placement']['availability-zone'] + except KeyError: + pass + if not zone and spotBid: + # if spot bid is present, all the other parameters must be as well + assert bool(spotBid) == bool(nodeType) == bool(ctx) + # if the zone is unset and we are using the spot market, optimize our + # choice based on the spot history + return optimize_spot_bid(ctx=ctx, instance_type=nodeType, spot_bid=float(spotBid)) + if not zone: + zone = boto.config.get('Boto', 'ec2_region_name') + if zone is not None: + zone += 'a' # derive an availability zone in the region + return zone @@ -201,55 +197,3 @@ def _get_spot_history(ctx, instance_type): product_description="Linux/UNIX") spot_data.sort(key=attrgetter("timestamp"), reverse=True) return spot_data - - -def checkValidNodeTypes(provisioner, nodeTypes): - """ - Raises if an invalid nodeType is specified for aws or gce. - - :param str provisioner: 'aws' or 'gce' to specify which cloud provisioner used. - :param nodeTypes: A list of node types. Example: ['t2.micro', 't2.medium'] - :return: Nothing. Raises if invalid nodeType. - """ - # TODO: Move out of "aws.__init__.py" >.> - if not nodeTypes: - return - if not isinstance(nodeTypes, list): - nodeTypes = [nodeTypes] - if not isinstance(nodeTypes[0], str): - return - # check if a valid node type for aws - from toil.lib.generatedEC2Lists import E2Instances, regionDict - if provisioner == 'aws': - from toil.provisioners.aws import getCurrentAWSZone - currentZone = getCurrentAWSZone() - if not currentZone: - currentZone = 'us-west-2' - else: - currentZone = currentZone[:-1] # adds something like 'a' or 'b' to the end - # check if instance type exists in this region - for nodeType in nodeTypes: - if nodeType and ':' in nodeType: - nodeType = nodeType.split(':')[0] - if nodeType not in regionDict[currentZone]: - # They probably misspelled it and can't tell. - close = get_close_matches(nodeType, regionDict[currentZone], 1) - if len(close) > 0: - helpText = ' Did you mean ' + close[0] + '?' - else: - helpText = '' - raise RuntimeError('Invalid nodeType (%s) specified for AWS in region: %s.%s' - '' % (nodeType, currentZone, helpText)) - elif provisioner == 'gce': - for nodeType in nodeTypes: - if nodeType and ':' in nodeType: - nodeType = nodeType.split(':')[0] - try: - E2Instances[nodeType] - raise RuntimeError("It looks like you've specified an AWS nodeType with the " - "{} provisioner. Please specify an {} nodeType." - "".format(provisioner, provisioner)) - except KeyError: - pass - else: - raise RuntimeError("Invalid provisioner: {}".format(provisioner)) diff --git a/src/toil/provisioners/aws/awsProvisioner.py b/src/toil/provisioners/aws/awsProvisioner.py index 1b9967ba97..f6af612cc2 100644 --- a/src/toil/provisioners/aws/awsProvisioner.py +++ b/src/toil/provisioners/aws/awsProvisioner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 UCSC Computational Genomics Lab # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,16 +26,19 @@ from boto.utils import get_instance_metadata from toil.lib.context import Context -from toil.lib.ec2 import (a_short_time, create_instances, - create_ondemand_instances, create_spot_instances, - wait_instances_running, wait_transition) +from toil.lib.ec2 import (a_short_time, + create_instances, + create_ondemand_instances, + create_spot_instances, + wait_instances_running, + wait_transition) from toil.lib.generatedEC2Lists import E2Instances -from toil.lib.memoize import less_strict_bool, memoize +from toil.lib.memoize import memoize from toil.lib.misc import truncExpBackoff from toil.lib.retry import old_retry from toil.provisioners import NoSuchClusterException from toil.provisioners.abstractProvisioner import AbstractProvisioner, Shape -from toil.provisioners.aws import getCurrentAWSZone, getSpotZone, zoneToRegion +from toil.provisioners.aws import get_current_aws_zone, zone_to_region from toil.provisioners.node import Node logger = logging.getLogger(__name__) @@ -66,7 +69,7 @@ def awsRetryPredicate(e): def awsFilterImpairedNodes(nodes, ec2): # if TOIL_AWS_NODE_DEBUG is set don't terminate nodes with # failing status checks so they can be debugged - nodeDebug = less_strict_bool(os.environ.get('TOIL_AWS_NODE_DEBUG')) + nodeDebug = os.environ.get('TOIL_AWS_NODE_DEBUG') in ('True', 'TRUE', 'true', True) if not nodeDebug: return nodes nodeIDs = [node.id for node in nodes] @@ -75,7 +78,7 @@ def awsFilterImpairedNodes(nodes, ec2): healthyNodes = [node for node in nodes if statusMap.get(node.id, None) != 'impaired'] impairedNodes = [node.id for node in nodes if statusMap.get(node.id, None) == 'impaired'] logger.warning('TOIL_AWS_NODE_DEBUG is set and nodes %s have failed EC2 status checks so ' - 'will not be terminated.', ' '.join(impairedNodes)) + 'will not be terminated.', ' '.join(impairedNodes)) return healthyNodes @@ -104,10 +107,10 @@ def __init__(self, clusterName, zone, nodeStorage, nodeStorageOverrides, sseKey) super(AWSProvisioner, self).__init__(clusterName, zone, nodeStorage, nodeStorageOverrides) self.cloud = 'aws' self._sseKey = sseKey - self._zone = zone if zone else getCurrentAWSZone() + self._zone = zone if zone else get_current_aws_zone() # establish boto3 clients - self.session = boto3.Session(region_name=zoneToRegion(self._zone)) + self.session = boto3.Session(region_name=zone_to_region(self._zone)) self.ec2 = self.session.resource('ec2') if clusterName: @@ -121,7 +124,7 @@ def _readClusterSettings(self): is the leader. """ instanceMetaData = get_instance_metadata() - region = zoneToRegion(self._zone) + region = zone_to_region(self._zone) conn = boto.ec2.connect_to_region(region) instance = conn.get_all_instances(instance_ids=[instanceMetaData["instance-id"]])[0].instances[0] self.clusterName = str(instance.tags["Name"]) @@ -343,7 +346,7 @@ def addNodes(self, nodeType, numNodes, preemptable, spotBid=None): spec=kwargs, num_instances=numNodes) else: logger.debug('Launching %s preemptable nodes', numNodes) - kwargs['placement'] = getSpotZone(spotBid, instanceType.name, self._ctx) + kwargs['placement'] = get_current_aws_zone(spotBid, instanceType.name, self._ctx) # force generator to evaluate instancesLaunched = list(create_spot_instances(ec2=self._ctx.ec2, price=spotBid, @@ -389,7 +392,7 @@ def getProvisionedWorkers(self, nodeType, preemptable): def _buildContext(self): if self._zone is None: - self._zone = getCurrentAWSZone() + self._zone = get_current_aws_zone() if self._zone is None: raise RuntimeError( 'Could not determine availability zone. Ensure that one of the following ' @@ -420,7 +423,7 @@ def _discoverAMI(self): JSON_FEED_URL = 'https://stable.release.flatcar-linux.net/amd64-usr/current/flatcar_production_ami_all.json' # What region do we care about? - region = zoneToRegion(self._zone) + region = zone_to_region(self._zone) for attempt in old_retry(predicate=lambda e: True): # Until we get parseable JSON diff --git a/src/toil/provisioners/clusterScaler.py b/src/toil/provisioners/clusterScaler.py index 0ccb77f5bd..b951ac2ac1 100644 --- a/src/toil/provisioners/clusterScaler.py +++ b/src/toil/provisioners/clusterScaler.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - import json import logging import os @@ -31,6 +29,7 @@ logger = logging.getLogger(__name__) + class BinPackedFit(object): """ If jobShapes is a set of tasks with run requirements (mem/disk/cpu), and nodeShapes is a sorted diff --git a/src/toil/provisioners/gceProvisioner.py b/src/toil/provisioners/gceProvisioner.py index 36a4741702..e9b7804fb5 100644 --- a/src/toil/provisioners/gceProvisioner.py +++ b/src/toil/provisioners/gceProvisioner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/provisioners/node.py b/src/toil/provisioners/node.py index 8261c1c459..7b6d171c41 100644 --- a/src/toil/provisioners/node.py +++ b/src/toil/provisioners/node.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/realtimeLogger.py b/src/toil/realtimeLogger.py index 1bcf14265b..dd9b9b7057 100644 --- a/src/toil/realtimeLogger.py +++ b/src/toil/realtimeLogger.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,11 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -""" -Implements a real-time UDP-based logging system that user scripts can use for debugging. -""" - +"""Implements a real-time UDP-based logging system that user scripts can use for debugging.""" import json import logging import logging.handlers @@ -24,10 +20,10 @@ import socketserver as SocketServer import threading -import toil.lib.bioio from toil.batchSystems.options import getPublicIP +from toil.statsAndLogging import set_log_level -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) class LoggingDatagramHandler(SocketServer.BaseRequestHandler): @@ -68,7 +64,7 @@ def handle(self): else: # Log level filtering should have been done on the remote end. The handle() method # skips it on this end. - log.handle(record) + logger.handle(record) class JSONDatagramHandler(logging.handlers.DatagramHandler): @@ -135,7 +131,7 @@ def _startLeader(cls, batchSystem, level=defaultLevel): if cls.initialized == 0: cls.initialized += 1 if level: - log.info('Starting real-time logging.') + logger.info('Starting real-time logging.') # Start up the logging server cls.loggingServer = SocketServer.ThreadingUDPServer( server_address=('0.0.0.0', 0), @@ -158,10 +154,10 @@ def _setEnv(name, value): _setEnv('ADDRESS', '%s:%i' % (ip, port)) _setEnv('LEVEL', level) else: - log.debug('Real-time logging disabled') + logger.debug('Real-time logging disabled') else: if level: - log.warning('Ignoring nested request to start real-time logging') + logger.warning('Ignoring nested request to start real-time logging') @classmethod def _stopLeader(cls): @@ -173,11 +169,11 @@ def _stopLeader(cls): cls.initialized -= 1 if cls.initialized == 0: if cls.loggingServer: - log.info('Stopping real-time logging server.') + logger.info('Stopping real-time logging server.') cls.loggingServer.shutdown() cls.loggingServer = None if cls.serverThread: - log.info('Joining real-time logging server thread.') + logger.info('Joining real-time logging server thread.') cls.serverThread.join() cls.serverThread = None for k in list(os.environ.keys()): @@ -188,7 +184,7 @@ def _stopLeader(cls): def getLogger(cls): """ Get the logger that logs real-time to the leader. - + Note that if the returned logger is used on the leader, you will see the message twice, since it still goes to the normal log handlers, too. """ @@ -207,7 +203,7 @@ def getLogger(cls): cls.logger.setLevel(logging.CRITICAL) else: # Adopt the logging level set on the leader. - toil.lib.bioio.setLogLevel(level, cls.logger) + set_log_level(level, cls.logger) try: address = os.environ[cls.envPrefix + 'ADDRESS'] except KeyError: diff --git a/src/toil/resource.py b/src/toil/resource.py index 5acc6a98d6..99603d66f9 100644 --- a/src/toil/resource.py +++ b/src/toil/resource.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ from toil.lib.retry import ErrorCondition, retry from toil.version import exactPython -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) class Resource(namedtuple('Resource', ('name', 'pathHash', 'url', 'contentHash'))): @@ -291,10 +291,10 @@ def _load(cls, path): fullPath = os.path.join(dirName, fileName) zipFile.write(fullPath, os.path.relpath(fullPath, rootDir)) except IOError: - log.critical('Cannot access and read the file at path: %s' % fullPath) + logger.critical('Cannot access and read the file at path: %s' % fullPath) sys.exit(1) else: - log.critical("Couldn't package the directory at %s for hot deployment. Would recommend to create a \ + logger.critical("Couldn't package the directory at %s for hot deployment. Would recommend to create a \ subdirectory (ie %s/MYDIR_HERE/)" % (path, path)) sys.exit(1) bytesIO.seek(0) @@ -390,11 +390,11 @@ def forModule(cls, name): if not extension in ('.py', '.pyc'): raise Exception('The name of a user script/module must end in .py or .pyc.') if name == '__main__': - log.debug("Discovering real name of module") + logger.debug("Discovering real name of module") # User script/module was invoked as the main program if module.__package__: # Invoked as a module via python -m foo.bar - log.debug("Script was invoked as a module") + logger.debug("Script was invoked as a module") name = [filePath.pop()] for package in reversed(module.__package__.split('.')): dirPathTail = filePath.pop() @@ -419,7 +419,7 @@ def forModule(cls, name): dirPath = os.path.abspath(os.path.sep.join(filePath)) absPrefix = os.path.abspath(sys.prefix) inVenv = inVirtualEnv() - log.debug("Module dir is %s, our prefix is %s, virtualenv: %s", dirPath, absPrefix, inVenv) + logger.debug("Module dir is %s, our prefix is %s, virtualenv: %s", dirPath, absPrefix, inVenv) if not os.path.isdir(dirPath): raise Exception('Bad directory path %s for module %s. Note that hot-deployment does not support .egg-link files yet, or scripts located in the root directory.' % (dirPath, name)) fromVirtualEnv = inVenv and dirPath.startswith(absPrefix) @@ -490,7 +490,7 @@ def localize(self): :rtype: toil.resource.Resource """ if not self._runningOnWorker(): - log.warning('The localize() method should only be invoked on a worker.') + logger.warning('The localize() method should only be invoked on a worker.') resource = Resource.lookup(self._resourcePath) if resource is None: return self @@ -510,7 +510,7 @@ def _runningOnWorker(self): try: mainModule = sys.modules['__main__'] except KeyError: - log.warning('Cannot determine main program module.') + logger.warning('Cannot determine main program module.') return False else: # If __file__ is not a valid attribute, it's because @@ -522,8 +522,7 @@ def _runningOnWorker(self): except AttributeError: return False - workerModuleFiles = concat(('worker' + ext for ext in self.moduleExtensions), - '_toil_worker') # the setuptools entry point + workerModuleFiles = ['worker.py', 'worker.pyc', 'worker.pyo', '_toil_worker'] # setuptools entry point return mainModuleFile in workerModuleFiles def globalize(self): @@ -564,12 +563,9 @@ def _resourcePath(self): tuple(concat(initName, self.dirPath, exactPython, os.path.split(self.dirPath), self.name))) return self.dirPath - moduleExtensions = ('.py', '.pyc', '.pyo') - @classmethod def _initModuleName(cls, dirPath): - for extension in cls.moduleExtensions: - name = '__init__' + extension + for name in ('__init__.py', '__init__.pyc', '__init__.pyo'): if os.path.exists(os.path.join(dirPath, name)): return name return None @@ -601,7 +597,7 @@ def load(self): try: return importlib.import_module(module.name) except ImportError: - log.error('Failed to import user module %r from sys.path (%r).', module, sys.path) + logger.error('Failed to import user module %r from sys.path (%r).', module, sys.path) raise diff --git a/src/toil/serviceManager.py b/src/toil/serviceManager.py index 19a37e9d26..7a56f31e92 100644 --- a/src/toil/serviceManager.py +++ b/src/toil/serviceManager.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/statsAndLogging.py b/src/toil/statsAndLogging.py index 79cc33d801..64e4e7370e 100644 --- a/src/toil/statsAndLogging.py +++ b/src/toil/statsAndLogging.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,26 +11,28 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - import gzip import json import logging import os import time +from argparse import ArgumentParser from threading import Event, Thread +from typing import Optional, Union, TextIO, BinaryIO -from toil.lib.bioio import getTotalCpuTime from toil.lib.expando import Expando +from toil.lib.resources import get_total_cpu_time -logger = logging.getLogger( __name__ ) +logger = logging.getLogger(__name__) +root_logger = logging.getLogger() +toil_logger = logging.getLogger('toil') +DEFAULT_LOGLEVEL = logging.INFO +__loggingFiles = [] -class StatsAndLogging( object ): - """ - Class manages a thread that aggregates statistics and logging information on a toil run. - """ +class StatsAndLogging: + """A thread to aggregate statistics and logging.""" def __init__(self, jobStore, config): self._stop = Event() self._worker = Thread(target=self.statsAndLoggingAggregator, @@ -38,43 +40,26 @@ def __init__(self, jobStore, config): daemon=True) def start(self): - """ - Start the stats and logging thread. - """ + """Start the stats and logging thread.""" self._worker.start() @classmethod - def formatLogStream(cls, stream, identifier=None): + def formatLogStream(cls, stream: Union[TextIO, BinaryIO], job_name: Optional[str] = None): """ Given a stream of text or bytes, and the job name, job itself, or some other optional stringifyable identity info for the job, return a big text string with the formatted job log, suitable for printing for the user. - + We don't want to prefix every line of the job's log with our own logging info, or we get prefixes wider than any reasonable terminal and longer than the messages. """ - - lines = [] - - if identifier is not None: - if isinstance(identifier, bytes): - # Decode the identifier if it is bytes - identifier = identifier.decode('utf-8', error='replace') - elif not isinstance(identifier, str): - # Otherwise just stringify it - identifier = str(identifier) - - lines.append('Log from job %s follows:' % identifier) - else: - lines.append('Log from job follows:') - - lines.append('=========>') - + lines = [f'Log from job "{job_name}" follows:', '=========>'] + for line in stream: if isinstance(line, bytes): - line = line.decode('utf-8') + line = line.decode('utf-8', errors='replace') lines.append('\t' + line.rstrip('\n')) lines.append('<=========') @@ -86,7 +71,7 @@ def formatLogStream(cls, stream, identifier=None): def logWithFormatting(cls, jobStoreID, jobLogs, method=logger.debug, message=None): if message is not None: method(message) - + # Format and log the logs, identifying the job with its job store ID. method(cls.formatLogStream(jobLogs, jobStoreID)) @@ -128,10 +113,8 @@ def createName(logPath, jobName, logExtension, failed=False): fullName = createName(path, mainFileName, extension, failed) with writeFn(fullName, 'wb') as f: for l in jobLogList: - try: + if isinstance(l, bytes): l = l.decode('utf-8') - except AttributeError: - pass if not l.endswith('\n'): l += '\n' f.write(l.encode('utf-8')) @@ -150,7 +133,7 @@ def statsAndLoggingAggregator(cls, jobStore, stop, config): """ # Overall timing startTime = time.time() - startClock = getTotalCpuTime() + startClock = get_total_cpu_time() def callback(fileHandle): statsStr = fileHandle.read() @@ -189,7 +172,7 @@ def callback(fileHandle): # Finish the stats file text = json.dumps(dict(total_time=str(time.time() - startTime), - total_clock=str(getTotalCpuTime() - startClock)), ensure_ascii=True) + total_clock=str(get_total_cpu_time() - startClock)), ensure_ascii=True) jobStore.writeStatsAndLogging(text) def check(self): @@ -201,12 +184,102 @@ def check(self): raise RuntimeError("Stats and logging thread has quit") def shutdown(self): - """ - Finish up the stats/logging aggregation thread - """ + """Finish up the stats/logging aggregation thread.""" logger.debug('Waiting for stats and logging collator thread to finish ...') startTime = time.time() self._stop.set() self._worker.join() logger.debug('... finished collating stats and logs. Took %s seconds', time.time() - startTime) # in addition to cleaning on exceptions, onError should clean if there are any failed jobs + + +def set_log_level(level, set_logger=None): + """Sets the root logger level to a given string level (like "INFO").""" + level = "CRITICAL" if level.upper() == "OFF" else level.upper() + set_logger = set_logger if set_logger else root_logger + set_logger.setLevel(level) + + # Suppress any random loggers introduced by libraries we use. + # Especially boto/boto3. They print too much. -__- + suppress_exotic_logging(__name__) + + +def add_logging_options(parser: ArgumentParser): + """Add logging options to set the global log level.""" + group = parser.add_argument_group("Logging Options") + default_loglevel = logging.getLevelName(DEFAULT_LOGLEVEL) + + levels = ['Critical', 'Error', 'Warning', 'Debug', 'Info'] + for level in levels: + group.add_argument(f"--log{level}", dest="logLevel", default=default_loglevel, action="store_const", + const=level, help=f"Turn on loglevel {level}. Default: {default_loglevel}.") + + levels += [l.lower() for l in levels] + [l.upper() for l in levels] + group.add_argument("--logOff", dest="logLevel", default=default_loglevel, + action="store_const", const="CRITICAL", help="Same as --logCRITICAL.") + group.add_argument("--logLevel", dest="logLevel", default=default_loglevel, choices=levels, + help=f"Set the log level. Default: {default_loglevel}. Options: {levels}.") + group.add_argument("--logFile", dest="logFile", help="File to log in.") + group.add_argument("--rotatingLogging", dest="logRotating", action="store_true", default=False, + help="Turn on rotating logging, which prevents log files from getting too big.") + + +def configure_root_logger(): + """ + Set up the root logger with handlers and formatting. + + Should be called before any entry point tries to log anything, + to ensure consistent formatting. + """ + logging.basicConfig(format='[%(asctime)s] [%(threadName)-10s] [%(levelname).1s] [%(name)s] %(message)s', + datefmt='%Y-%m-%dT%H:%M:%S%z') + root_logger.setLevel(DEFAULT_LOGLEVEL) + + +def log_to_file(log_file, log_rotation): + if log_file and log_file not in __loggingFiles: + logger.debug(f"Logging to file '{log_file}'.") + __loggingFiles.append(log_file) + if log_rotation: + handler = logging.handlers.RotatingFileHandler(log_file, maxBytes=1000000, backupCount=1) + else: + handler = logging.FileHandler(log_file) + root_logger.addHandler(handler) + + +def set_logging_from_options(options): + configure_root_logger() + options.logLevel = options.logLevel or logging.getLevelName(root_logger.getEffectiveLevel()) + set_log_level(options.logLevel) + logger.debug(f"Root logger is at level '{logging.getLevelName(root_logger.getEffectiveLevel())}', " + f"'toil' logger at level '{logging.getLevelName(toil_logger.getEffectiveLevel())}'.") + + # start logging to log file if specified + log_to_file(options.logFile, options.logRotating) + + +def suppress_exotic_logging(local_logger): + """ + Attempts to suppress the loggers of all non-Toil packages by setting them to CRITICAL. + + For example: 'requests_oauthlib', 'google', 'boto', 'websocket', 'oauthlib', etc. + + This will only suppress loggers that have already been instantiated and can be seen in the environment, + except for the list declared in "always_suppress". + + This is important because some packages, particularly boto3, are not always instantiated yet in the + environment when this is run, and so we create the logger and set the level preemptively. + """ + never_suppress = ['toil', '__init__', '__main__', 'toil-rt', 'cwltool'] + always_suppress = ['boto3', 'boto', 'botocore'] # ensure we suppress even before instantiated + + top_level_loggers = list() + for pkg_logger in list(logging.Logger.manager.loggerDict.keys()) + always_suppress: + if pkg_logger != local_logger: + # many sub-loggers may exist, like "boto.a", "boto.b", "boto.c"; we only want the top_level: "boto" + top_level_logger = pkg_logger.split('.')[0] if '.' in pkg_logger else pkg_logger + + if top_level_logger not in top_level_loggers + never_suppress: + top_level_loggers.append(top_level_logger) + logging.getLogger(top_level_logger).setLevel(logging.CRITICAL) + logger.debug(f'Suppressing the following loggers: {set(top_level_loggers)}') diff --git a/src/toil/test/__init__.py b/src/toil/test/__init__.py index 3108a71cc5..7e38f4fd06 100644 --- a/src/toil/test/__init__.py +++ b/src/toil/test/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ import datetime import logging import os +import random import re import shutil import signal @@ -37,11 +38,10 @@ from toil.lib.iterables import concat from toil.lib.memoize import memoize from toil.lib.threading import ExceptionalThread, cpu_count -from toil.provisioners.aws import runningOnEC2 +from toil.provisioners.aws import running_on_ec2 from toil.version import distVersion -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) class ToilTest(unittest.TestCase): @@ -88,12 +88,12 @@ def tearDownClass(cls): super(ToilTest, cls).tearDownClass() def setUp(self): - log.info("Setting up %s ...", self.id()) + logger.info("Setting up %s ...", self.id()) super(ToilTest, self).setUp() def tearDown(self): super(ToilTest, self).tearDown() - log.info("Tore down %s", self.id()) + logger.info("Tore down %s", self.id()) @classmethod def awsRegion(cls): @@ -101,7 +101,7 @@ def awsRegion(cls): Use us-west-2 unless running on EC2, in which case use the region in which the instance is located """ - return cls._region() if runningOnEC2() else 'us-west-2' + return cls._region() if running_on_ec2() else 'us-west-2' @classmethod def _availabilityZone(cls): @@ -205,7 +205,7 @@ def _run(cls, command, *args, **kwargs): :return: The output of the process' stdout if capture=True was passed, None otherwise. """ args = list(concat(command, args)) - log.info('Running %r', args) + logger.info('Running %r', args) capture = kwargs.pop('capture', False) _input = kwargs.pop('input', None) if capture: @@ -241,6 +241,20 @@ def _mark_test(name, test_item): return getattr(pytest.mark, name)(test_item) +def get_temp_file(suffix="", rootDir=None): + """Returns a string representing a temporary file, that must be manually deleted.""" + if rootDir is None: + handle, tmp_file = tempfile.mkstemp(suffix) + os.close(handle) + return tmp_file + else: + alphanumerics = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' + tmp_file = os.path.join(rootDir, f"tmp_{''.join([random.choice(alphanumerics) for _ in range(0, 10)])}{suffix}") + open(tmp_file, 'w').close() + os.chmod(tmp_file, 0o777) # Ensure everyone has access to the file. + return tmp_file + + def needs_rsync3(test_item): """ Use as a decorator before test classes or methods that depend on any features used in rsync @@ -269,7 +283,7 @@ def needs_aws_s3(test_item): except ImportError: return unittest.skip("Install Toil with the 'aws' extra to include this test.")(test_item) - if not (boto_credentials or os.path.exists(os.path.expanduser('~/.aws/credentials')) or runningOnEC2()): + if not (boto_credentials or os.path.exists(os.path.expanduser('~/.aws/credentials')) or running_on_ec2()): return unittest.skip("Configure AWS credentials to include this test.")(test_item) return test_item @@ -345,7 +359,7 @@ def needs_mesos(test_item): import psutil import pymesos print(psutil.__file__) - pritn(pymesos.__file__) # keep these imports from being removed. + print(pymesos.__file__) # keep these imports from being removed. except ImportError: return unittest.skip("Install Mesos (and Toil with the 'mesos' extra) to include this test.")(test_item) return test_item @@ -684,27 +698,6 @@ def fx(self, prms=prms): insertMethodToClass() -@contextmanager -def tempFileContaining(content, suffix=''): - """ - Write a file with the given contents, and keep it on disk as long as the context is active. - :param str content: The contents of the file. - :param str suffix: The extension to use for the temporary file. - """ - fd, path = tempfile.mkstemp(suffix=suffix) - try: - encoded = content.encode('utf-8') - assert os.write(fd, encoded) == len(encoded) - except: - os.close(fd) - raise - else: - os.close(fd) - yield path - finally: - os.unlink(path) - - class ApplianceTestSupport(ToilTest): """ A Toil test that runs a user script on a minimal cluster of appliance containers, @@ -777,7 +770,7 @@ def __enter__(self): ['--volume=%s:%s' % mount for mount in self.mounts.items()], image, self._containerCommand())) - log.info('Running %r', args) + logger.info('Running %r', args) self.popen = subprocess.Popen(args) self.start() self.__wait_running() @@ -797,7 +790,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False # don't swallow exception def __wait_running(self): - log.info("Waiting for %s container process to appear. " + logger.info("Waiting for %s container process to appear. " "Expect to see 'Error: No such image or container'.", self._getRole()) while self.isAlive(): try: @@ -832,7 +825,7 @@ def __cleanMounts(self): def tryRun(self): self.popen.wait() - log.info('Exiting %s', self.__class__.__name__) + logger.info('Exiting %s', self.__class__.__name__) def runOnAppliance(self, *args, **kwargs): # Check if thread is still alive. Note that ExceptionalThread.join raises the diff --git a/src/toil/test/batchSystems/__init__.py b/src/toil/test/batchSystems/__init__.py index f7d9550f46..471fe7cd32 100644 --- a/src/toil/test/batchSystems/__init__.py +++ b/src/toil/test/batchSystems/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py index dcf05f594f..f48849f0d8 100644 --- a/src/toil/test/batchSystems/batchSystemTest.py +++ b/src/toil/test/batchSystems/batchSystemTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import fcntl import itertools import logging @@ -33,17 +32,26 @@ # protected by annotations. from toil.batchSystems.mesos.test import MesosTestSupport from toil.batchSystems.parasol import ParasolBatchSystem -from toil.batchSystems.parasolTestSupport import ParasolTestSupport +from toil.test.batchSystems.parasolTestSupport import ParasolTestSupport from toil.batchSystems.singleMachine import SingleMachineBatchSystem from toil.common import Config from toil.job import Job, JobDescription from toil.lib.threading import cpu_count -from toil.test import (ToilTest, needs_aws_s3, needs_fetchable_appliance, - needs_gridengine, needs_htcondor, needs_kubernetes, - needs_lsf, needs_mesos, needs_parasol, needs_slurm, - needs_torque, slow, travis_test) - -log = logging.getLogger(__name__) +from toil.test import (ToilTest, + needs_aws_s3, + needs_fetchable_appliance, + needs_gridengine, + needs_htcondor, + needs_kubernetes, + needs_lsf, + needs_mesos, + needs_parasol, + needs_slurm, + needs_torque, + slow, + travis_test) + +logger = logging.getLogger(__name__) # How many cores should be utilized by this test. The test will fail if the running system # doesn't have at least that many cores. @@ -180,7 +188,7 @@ def testRunJobs(self): jobUpdateInfo = self.batchSystem.getUpdatedBatchJob(maxWait=1000) jobID, exitStatus, wallTime = jobUpdateInfo.jobID, jobUpdateInfo.exitStatus, jobUpdateInfo.wallTime - log.info('Third job completed: {} {} {}'.format(jobID, exitStatus, wallTime)) + logger.info('Third job completed: {} {} {}'.format(jobID, exitStatus, wallTime)) # Since the first two jobs were killed, the only job in the updated jobs queue should # be job 3. If the first two jobs were (incorrectly) added to the queue, this will @@ -268,7 +276,7 @@ def _waitForJobsToStart(self, numJobs, tries=20): # prevent an endless loop, give it a few tries for it in range(tries): running = self.batchSystem.getRunningBatchJobIDs() - log.info('Running jobs now: {}'.format(running)) + logger.info('Running jobs now: {}'.format(running)) runningIDs = list(running.keys()) if len(runningIDs) == numJobs: break @@ -540,7 +548,7 @@ def test(self): bs.shutdown() concurrentTasks, maxConcurrentTasks = getCounters(self.counterPath) self.assertEqual(concurrentTasks, 0) - log.info('maxCores: {maxCores}, ' + logger.info('maxCores: {maxCores}, ' 'coresPerJob: {coresPerJob}, ' 'load: {load}'.format(**locals())) # This is the key assertion: @@ -557,7 +565,7 @@ def testServices(self): Job.Runner.startToil(Job.wrapJobFn(parentJob, self.scriptCommand()), options) with open(self.counterPath, 'r+') as f: s = f.read() - log.info('Counter is %s', s) + logger.info('Counter is %s', s) self.assertEqual(getCounters(self.counterPath), (0, 3)) @@ -939,12 +947,12 @@ def count(delta, file_path): fcntl.flock(fd, fcntl.LOCK_EX) try: s = os.read(fd, 10) - value, maxValue = list(map(int, s.decode('utf-8').split(','))) + value, maxValue = [int(i) for i in s.decode('utf-8').split(',')] value += delta if value > maxValue: maxValue = value os.lseek(fd, 0, 0) os.ftruncate(fd, 0) - os.write(fd, ','.join(map(str, (value, maxValue))).encode('utf-8')) + os.write(fd, f'{value},{maxValue}'.encode('utf-8')) finally: fcntl.flock(fd, fcntl.LOCK_UN) finally: @@ -954,8 +962,7 @@ def count(delta, file_path): def getCounters(path): with open(path, 'r+') as f: - s = f.read() - concurrentTasks, maxConcurrentTasks = list(map(int, s.split(','))) + concurrentTasks, maxConcurrentTasks = [int(i) for i in f.read().split(',')] return concurrentTasks, maxConcurrentTasks diff --git a/src/toil/batchSystems/parasolTestSupport.py b/src/toil/test/batchSystems/parasolTestSupport.py similarity index 91% rename from src/toil/batchSystems/parasolTestSupport.py rename to src/toil/test/batchSystems/parasolTestSupport.py index f9f8d9644e..7119f3a737 100644 --- a/src/toil/batchSystems/parasolTestSupport.py +++ b/src/toil/test/batchSystems/parasolTestSupport.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import errno import logging import os import signal @@ -27,15 +25,6 @@ log = logging.getLogger(__name__) -def rm_f(path): - """Remove the file at the given path with os.remove(), ignoring errors caused by the file's absence.""" - try: - os.remove(path) - except OSError as e: - if e.errno == errno.ENOENT: - pass - else: - raise class ParasolTestSupport(object): """ @@ -63,7 +52,8 @@ def _stopParasol(self): self.leader.popen.kill() self.leader.join() for path in ('para.results', 'parasol.jid'): - rm_f(path) + if os.path.exists(path): + os.remove(path) class ParasolThread(threading.Thread): diff --git a/src/toil/test/cwl/conftest.py b/src/toil/test/cwl/conftest.py index 924b6e9d9d..813003b119 100644 --- a/src/toil/test/cwl/conftest.py +++ b/src/toil/test/cwl/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/cwl/cwlTest.py b/src/toil/test/cwl/cwlTest.py index 0403650538..380d9a7dc4 100644 --- a/src/toil/test/cwl/cwlTest.py +++ b/src/toil/test/cwl/cwlTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # Copyright (C) 2015 Curoverse, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,9 +31,17 @@ pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) # noqa sys.path.insert(0, pkg_root) # noqa -from toil.test import (ToilTest, needs_cwl, slow, needs_docker, needs_lsf, - needs_mesos, needs_parasol, needs_gridengine, needs_slurm, - needs_torque, needs_aws_s3) +from toil.test import (ToilTest, + needs_aws_s3, + needs_cwl, + needs_docker, + needs_gridengine, + needs_lsf, + needs_mesos, + needs_parasol, + needs_slurm, + needs_torque, + slow) log = logging.getLogger(__name__) CONFORMANCE_TEST_TIMEOUT = 3600 diff --git a/src/toil/test/docs/scripts/example_alwaysfail.py b/src/toil/test/docs/scripts/example_alwaysfail.py index b17bf4a656..33ca7b7172 100644 --- a/src/toil/test/docs/scripts/example_alwaysfail.py +++ b/src/toil/test/docs/scripts/example_alwaysfail.py @@ -35,5 +35,3 @@ def explode(job): if __name__=="__main__": main() - - diff --git a/src/toil/test/docs/scripts/example_cachingbenchmark.py b/src/toil/test/docs/scripts/example_cachingbenchmark.py index 7ffd9ba5c6..870c857ccc 100755 --- a/src/toil/test/docs/scripts/example_cachingbenchmark.py +++ b/src/toil/test/docs/scripts/example_cachingbenchmark.py @@ -103,6 +103,3 @@ def report(job, views): if __name__=="__main__": main() - - - diff --git a/src/toil/test/docs/scriptsTest.py b/src/toil/test/docs/scriptsTest.py index c7b18da349..4354206a6d 100644 --- a/src/toil/test/docs/scriptsTest.py +++ b/src/toil/test/docs/scriptsTest.py @@ -1,15 +1,14 @@ import os import re import shutil +import subprocess import sys import unittest pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) # noqa sys.path.insert(0, pkg_root) # noqa -import subprocess - -from toil.test import ToilTest, needs_cwl +from toil.test import ToilTest, needs_cwl, needs_docker, travis_test from toil.version import python @@ -17,9 +16,11 @@ class ToilDocumentationTest(ToilTest): """Tests for scripts in the toil tutorials.""" @classmethod def setUpClass(cls): + super(ToilTest, cls).setUpClass() cls.directory = os.path.dirname(os.path.abspath(__file__)) def tearDown(self): + super(ToilTest, self).tearDown() # src/toil/test/docs/scripts/cwlExampleFiles/sample_1_output.txt output_files = ["sample_1_output.txt", "sample_2_output.txt", "sample_3_output.txt"] for output in output_files: @@ -66,79 +67,102 @@ def checkExpectedPattern(self, script, expectedPattern): def testCwlexample(self): self.checkExitCode("tutorial_cwlexample.py") + @travis_test def testDiscoverfiles(self): self.checkExitCode("tutorial_discoverfiles.py") + @travis_test def testDynamic(self): self.checkExitCode("tutorial_dynamic.py") + @travis_test def testEncapsulation(self): self.checkExitCode("tutorial_encapsulation.py") + @travis_test def testEncapsulation2(self): self.checkExitCode("tutorial_encapsulation2.py") + @travis_test def testHelloworld(self): self.checkExpectedOut("tutorial_helloworld.py", "Hello, world!, here's a message: You did it!\n") + @travis_test def testInvokeworkflow(self): self.checkExpectedOut("tutorial_invokeworkflow.py", "Hello, world!, here's a message: Woot\n") + @travis_test def testInvokeworkflow2(self): self.checkExpectedOut("tutorial_invokeworkflow2.py", "Hello, world!, I have a message: Woot!\n") + @travis_test def testJobFunctions(self): self.checkExpectedOut("tutorial_jobfunctions.py", "Hello world, I have a message: Woot!\n") + @travis_test def testManaging(self): self.checkExitCode("tutorial_managing.py") + @travis_test def testManaging2(self): self.checkExitCode("tutorial_managing2.py") + @travis_test def testMultiplejobs(self): - self.checkExpectedPattern("tutorial_multiplejobs.py", "Hello world, I have a message: first.*Hello world, I have a message: " - "second or third.*Hello world, I have a message: second or third.*Hello world," - " I have a message: last") + self.checkExpectedPattern("tutorial_multiplejobs.py", + "Hello world, I have a message: first.*Hello world, I have a message: " + "second or third.*Hello world, I have a message: second or third.*Hello world," + " I have a message: last") + @travis_test def testMultiplejobs2(self): - self.checkExpectedPattern("tutorial_multiplejobs2.py", "Hello world, I have a message: first.*Hello world, I have a message: " - "second or third.*Hello world, I have a message: second or third.*Hello world," - " I have a message: last") + self.checkExpectedPattern("tutorial_multiplejobs2.py", + "Hello world, I have a message: first.*Hello world, I have a message: " + "second or third.*Hello world, I have a message: second or third.*Hello world," + " I have a message: last") + @travis_test def testMultiplejobs3(self): - self.checkExpectedPattern("tutorial_multiplejobs3.py", "Hello world, I have a message: first.*Hello world, I have a message: " - "second or third.*Hello world, I have a message: second or third.*Hello world," - " I have a message: last") + self.checkExpectedPattern("tutorial_multiplejobs3.py", + "Hello world, I have a message: first.*Hello world, I have a message: " + "second or third.*Hello world, I have a message: second or third.*Hello world," + " I have a message: last") + @travis_test def testPromises2(self): - self.checkExpectedOut("tutorial_promises2.py", "['00000', '00001', '00010', '00011', '00100', '00101', '00110', '00111', '01000'," - " '01001', '01010', '01011', '01100', '01101', '01110', '01111', '10000', '10001', " - "'10010', '10011', '10100', '10101', '10110', '10111', '11000', '11001', '11010', " - "'11011', '11100', '11101', '11110', '11111']") + self.checkExpectedOut("tutorial_promises2.py", + "['00000', '00001', '00010', '00011', '00100', '00101', '00110', '00111'," + " '01000', '01001', '01010', '01011', '01100', '01101', '01110', '01111'," + " '10000', '10001', '10010', '10011', '10100', '10101', '10110', '10111'," + " '11000', '11001', '11010', '11011', '11100', '11101', '11110', '11111']") + @travis_test def testQuickstart(self): self.checkExpectedOut("tutorial_quickstart.py", "Hello, world!, here's a message: Woot\n") + @travis_test def testRequirements(self): self.checkExitCode("tutorial_requirements.py") + @travis_test def testArguments(self): self.checkExpectedOut("tutorial_arguments.py", "Hello, world!, here's a message: Woot") - """@needs_docker # timing out; likely need to update docker on toil + @needs_docker def testDocker(self): - self.checkExitCode("tutorial_docker.py")""" + self.checkExitCode("tutorial_docker.py") + @travis_test def testPromises(self): self.checkExpectedPattern("tutorial_promises.py", "i is: 1.*i is: 2.*i is: 3") + @travis_test def testServices(self): self.checkExitCode("tutorial_services.py") - """Needs cromwell jar file to run + @unittest.skip('Needs cromwell jar file to run.') def testWdlexample(self): - self.checkExitCode("tutorial_wdlexample.py")""" + self.checkExitCode("tutorial_wdlexample.py") if __name__ == "__main__": diff --git a/src/toil/test/jobStores/__init__.py b/src/toil/test/jobStores/__init__.py index f7d9550f46..471fe7cd32 100644 --- a/src/toil/test/jobStores/__init__.py +++ b/src/toil/test/jobStores/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py index 27c6e4c0f1..62bba488f7 100644 --- a/src/toil/test/jobStores/jobStoreTest.py +++ b/src/toil/test/jobStores/jobStoreTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -42,8 +42,13 @@ from toil.lib.exceptions import panic from toil.lib.memoize import memoize from toil.statsAndLogging import StatsAndLogging -from toil.test import (ToilTest, make_tests, needs_aws_s3, needs_encryption, - needs_google, slow, travis_test) +from toil.test import (ToilTest, + make_tests, + needs_aws_s3, + needs_encryption, + needs_google, + slow, + travis_test) # noinspection PyPackageRequirements # (installed by `make prepare`) diff --git a/src/toil/test/lib/dockerTest.py b/src/toil/test/lib/dockerTest.py index bcf2a80ea4..d2325c676c 100644 --- a/src/toil/test/lib/dockerTest.py +++ b/src/toil/test/lib/dockerTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,8 +22,12 @@ from docker.errors import ContainerError from toil.job import Job from toil.leader import FailedJobsException -from toil.lib.docker import (FORGO, RM, STOP, apiDockerCall, - containerIsRunning, dockerKill) +from toil.lib.docker import (FORGO, + RM, + STOP, + apiDockerCall, + containerIsRunning, + dockerKill) from toil.test import ToilTest, needs_docker, slow logger = logging.getLogger(__name__) diff --git a/src/toil/test/mesos/MesosDataStructuresTest.py b/src/toil/test/mesos/MesosDataStructuresTest.py index 136695611a..b0437f938c 100644 --- a/src/toil/test/mesos/MesosDataStructuresTest.py +++ b/src/toil/test/mesos/MesosDataStructuresTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/mesos/__init__.py b/src/toil/test/mesos/__init__.py index f7d9550f46..471fe7cd32 100644 --- a/src/toil/test/mesos/__init__.py +++ b/src/toil/test/mesos/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/mesos/helloWorld.py b/src/toil/test/mesos/helloWorld.py index 46b41450d5..c8e33c5b57 100644 --- a/src/toil/test/mesos/helloWorld.py +++ b/src/toil/test/mesos/helloWorld.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/mesos/stress.py b/src/toil/test/mesos/stress.py index 28d14663bb..ac74e6967d 100644 --- a/src/toil/test/mesos/stress.py +++ b/src/toil/test/mesos/stress.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/provisioners/__init__.py b/src/toil/test/provisioners/__init__.py index f7d9550f46..471fe7cd32 100644 --- a/src/toil/test/provisioners/__init__.py +++ b/src/toil/test/provisioners/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/provisioners/aws/__init__.py b/src/toil/test/provisioners/aws/__init__.py index 6579f75b98..471fe7cd32 100644 --- a/src/toil/test/provisioners/aws/__init__.py +++ b/src/toil/test/provisioners/aws/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/provisioners/aws/awsProvisionerTest.py b/src/toil/test/provisioners/aws/awsProvisionerTest.py index 0429b1b7d9..30e9bd9705 100644 --- a/src/toil/test/provisioners/aws/awsProvisionerTest.py +++ b/src/toil/test/provisioners/aws/awsProvisionerTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,10 +22,14 @@ import pytest -from toil.provisioners import clusterFactory +from toil.provisioners import cluster_factory from toil.provisioners.aws.awsProvisioner import AWSProvisioner -from toil.test import (ToilTest, integrative, needs_appliance, needs_aws_ec2, - slow, timeLimit) +from toil.test import (ToilTest, + integrative, + needs_appliance, + needs_aws_ec2, + slow, + timeLimit) from toil.version import exactPython log = logging.getLogger(__name__) @@ -117,7 +121,7 @@ def _test(self, preemptableJobs=False): self.launchCluster() # get the leader so we know the IP address - we don't need to wait since create cluster # already insures the leader is running - self.cluster = clusterFactory(provisioner='aws', clusterName=self.clusterName) + self.cluster = cluster_factory(provisioner='aws', clusterName=self.clusterName) self.leader = self.cluster.getLeader() self.sshUtil(['mkdir', '-p', self.scriptDir]) # hot deploy doesn't seem permitted to work in normal /tmp or /home @@ -246,7 +250,7 @@ def launchCluster(self): self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage), '--nodeTypes', ",".join(self.instanceTypes), '-w', ",".join(self.numWorkers), '--nodeStorage', str(self.requestedLeaderStorage)]) - self.cluster = clusterFactory(provisioner='aws', clusterName=self.clusterName) + self.cluster = cluster_factory(provisioner='aws', clusterName=self.clusterName) nodes = self.cluster._getNodesInCluster(both=True) nodes.sort(key=lambda x: x.launch_time) # assuming that leader is first @@ -343,7 +347,7 @@ def f0(job): with open(tempfile_path, 'w') as f: # use appliance ssh method instead of sshutil so we can specify input param f.write(script) - cluster = clusterFactory(provisioner='aws', clusterName=self.clusterName) + cluster = cluster_factory(provisioner='aws', clusterName=self.clusterName) leader = cluster.getLeader() self.sshUtil(['mkdir', '-p', self.scriptDir]) # hot deploy doesn't seem permitted to work in normal /tmp or /home leader.injectFile(tempfile_path, self.scriptName, 'toil_leader') @@ -413,7 +417,7 @@ def job(job, disk='10M', cores=1, memory='10M', preemptable=True): script = dedent('\n'.join(getsource(userScript).split('\n')[1:])) # use appliance ssh method instead of sshutil so we can specify input param - cluster = clusterFactory(provisioner='aws', clusterName=self.clusterName) + cluster = cluster_factory(provisioner='aws', clusterName=self.clusterName) leader = cluster.getLeader() leader.sshAppliance('tee', '/home/userScript.py', input=script) @@ -422,4 +426,3 @@ def _runScript(self, toilOptions): command = ['/home/venv/bin/python', '/home/userScript.py'] command.extend(toilOptions) self.sshUtil(command) - diff --git a/src/toil/test/provisioners/clusterScalerTest.py b/src/toil/test/provisioners/clusterScalerTest.py index 61aec6850c..b7b6dfc502 100644 --- a/src/toil/test/provisioners/clusterScalerTest.py +++ b/src/toil/test/provisioners/clusterScalerTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -32,8 +32,10 @@ from toil.job import JobDescription from toil.lib.humanize import human2bytes as h2b from toil.provisioners.abstractProvisioner import AbstractProvisioner, Shape -from toil.provisioners.clusterScaler import (BinPackedFit, ClusterScaler, - NodeReservation, ScalerThread) +from toil.provisioners.clusterScaler import (BinPackedFit, + ClusterScaler, + NodeReservation, + ScalerThread) from toil.provisioners.node import Node from toil.test import ToilTest, slow, travis_test diff --git a/src/toil/test/provisioners/gceProvisionerTest.py b/src/toil/test/provisioners/gceProvisionerTest.py index a18acf4806..310f9e4a0b 100644 --- a/src/toil/test/provisioners/gceProvisionerTest.py +++ b/src/toil/test/provisioners/gceProvisionerTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,8 +19,12 @@ import pytest -from toil.test import (ToilTest, integrative, needs_appliance, needs_google, - slow, timeLimit) +from toil.test import (ToilTest, + integrative, + needs_appliance, + needs_google, + slow, + timeLimit) from toil.version import exactPython log = logging.getLogger(__name__) @@ -340,4 +344,3 @@ def _runScript(self, toilOptions): @integrative def testAutoScaledCluster(self): self._test() - diff --git a/src/toil/test/sort/__init__.py b/src/toil/test/sort/__init__.py index f7d9550f46..471fe7cd32 100644 --- a/src/toil/test/sort/__init__.py +++ b/src/toil/test/sort/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/sort/restart_sort.py b/src/toil/test/sort/restart_sort.py index 6cef84890a..760e215617 100644 --- a/src/toil/test/sort/restart_sort.py +++ b/src/toil/test/sort/restart_sort.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/sort/sort.py b/src/toil/test/sort/sort.py index 04cc3e4edf..aff650484e 100755 --- a/src/toil/test/sort/sort.py +++ b/src/toil/test/sort/sort.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/sort/sortTest.py b/src/toil/test/sort/sortTest.py index 3c3474338b..70e224092d 100755 --- a/src/toil/test/sort/sortTest.py +++ b/src/toil/test/sort/sortTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,19 +22,29 @@ from toil import resolveEntryPoint from toil.batchSystems.mesos.test import MesosTestSupport -from toil.batchSystems.parasolTestSupport import ParasolTestSupport +from toil.test.batchSystems.parasolTestSupport import ParasolTestSupport from toil.common import Toil from toil.job import Job from toil.jobStores.abstractJobStore import (JobStoreExistsException, NoSuchJobStoreException) from toil.leader import FailedJobsException -from toil.lib.bioio import getLogLevelString -from toil.test import (ToilTest, needs_aws_ec2, needs_google, needs_gridengine, - needs_mesos, needs_parasol, needs_torque, slow) -from toil.test.sort.sort import (copySubRangeOfFile, getMidPoint, main, - makeFileToSort, merge, sort) - -log = logging.getLogger(__name__) +from toil.lib.bioio import root_logger +from toil.test import (ToilTest, + needs_aws_ec2, + needs_google, + needs_gridengine, + needs_mesos, + needs_parasol, + needs_torque, + slow) +from toil.test.sort.sort import (copySubRangeOfFile, + getMidPoint, + main, + makeFileToSort, + merge, + sort) + +logger = logging.getLogger(__name__) defaultLineLen = int(os.environ.get('TOIL_TEST_SORT_LINE_LEN', 10)) defaultLines = int(os.environ.get('TOIL_TEST_SORT_LINES', 10)) @@ -94,7 +104,7 @@ def _toilSort(self, jobStoreLocator, batchSystem, try: # Specify options options = Job.Runner.getDefaultOptions(jobStoreLocator) - options.logLevel = getLogLevelString() + options.logLevel = logging.getLevelName(root_logger.getEffectiveLevel()) options.retryCount = retryCount options.batchSystem = batchSystem options.clean = "never" @@ -234,8 +244,6 @@ def testFileParasol(self): finally: self._stopParasol() - # The following functions test the functions in the test - testNo = 5 def testSort(self): @@ -292,19 +300,16 @@ def testGetMidPoint(self): sorted_contents = f.read() fileSize = os.path.getsize(self.inputFile) midPoint = getMidPoint(self.inputFile, 0, fileSize) - print("the mid point is %i of a file of %i bytes" % (midPoint, fileSize)) + print(f"The mid point is {midPoint} of a file of {fileSize} bytes.") assert midPoint < fileSize assert sorted_contents[midPoint] == '\n' assert midPoint >= 0 - # Support methods - def _awsJobStore(self): - return 'aws:%s:sort-test-%s' % (self.awsRegion(), uuid4()) + return f'aws:{self.awsRegion()}:sort-test-{uuid4()}' def _googleJobStore(self): - projectID = os.getenv('TOIL_GOOGLE_PROJECTID') - return 'google:%s:sort-test-%s' % (projectID, str(uuid4())) + return f'google:{os.getenv("TOIL_GOOGLE_PROJECTID")}:sort-test-{uuid4()}' def _loadFile(self, path): with open(path, 'r') as f: diff --git a/src/toil/test/src/__init__.py b/src/toil/test/src/__init__.py index f7d9550f46..471fe7cd32 100644 --- a/src/toil/test/src/__init__.py +++ b/src/toil/test/src/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/src/autoDeploymentTest.py b/src/toil/test/src/autoDeploymentTest.py index 06e332153e..642a1d526f 100644 --- a/src/toil/test/src/autoDeploymentTest.py +++ b/src/toil/test/src/autoDeploymentTest.py @@ -1,13 +1,13 @@ -# coding=utf-8 import logging import subprocess +import time from contextlib import contextmanager from toil.lib.iterables import concat from toil.test import ApplianceTestSupport, needs_appliance, needs_mesos, slow from toil.version import exactPython -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) @needs_mesos @@ -245,14 +245,9 @@ def testDeferralWithConcurrentEncapsulation(self): """ with self._venvApplianceCluster() as (leader, worker): def userScript(): - import logging - import time - from toil.common import Toil from toil.job import Job - log = logging.getLogger(__name__) - def root(rootJob): def nullFile(): return rootJob.fileStore.jobStore.importFile('file:///dev/null') diff --git a/src/toil/test/src/checkpointTest.py b/src/toil/test/src/checkpointTest.py index 0e24158899..2e3f64bc60 100644 --- a/src/toil/test/src/checkpointTest.py +++ b/src/toil/test/src/checkpointTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2017 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/src/deferredFunctionTest.py b/src/toil/test/src/deferredFunctionTest.py index 8e7eb1c68d..b5c70d0507 100644 --- a/src/toil/test/src/deferredFunctionTest.py +++ b/src/toil/test/src/deferredFunctionTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,23 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - import os import signal import time +import psutil from abc import ABCMeta from uuid import uuid4 -from toil.fileStores.cachingFileStore import CachingFileStore from toil.job import Job from toil.leader import FailedJobsException from toil.lib.threading import cpu_count from toil.test import ToilTest, slow, travis_test -# Some tests take too long on the AWS jobstore and are unquitable for CI. They can be -# be run during manual tests by setting this to False. -testingIsAutomatic = True class DeferredFunctionTest(ToilTest, metaclass=ABCMeta): """ @@ -295,12 +290,13 @@ def _testNewJobsCanHandleOtherJobDeaths_B(job, files): time.sleep(0.5) # Get the pid of _testNewJobsCanHandleOtherJobDeaths_A and wait for it to truly be dead. with open(files[1], 'r') as fileHandle: - meeseeksPID = int(fileHandle.read()) - while CachingFileStore._pidExists(meeseeksPID): + pid = int(fileHandle.read()) + assert pid > 0 + while psutil.pid_exists(pid): time.sleep(0.5) # Now that we are convinced that_testNewJobsCanHandleOtherJobDeaths_A has died, we can # spawn the next job - return None + def _testNewJobsCanHandleOtherJobDeaths_C(job, files, expectedResult): """ @@ -336,4 +332,3 @@ def _deleteFileClassMethod(cls, nonLocalFile, nlf=None): os.remove(nonLocalFile) if nlf is not None: os.remove(nlf) - diff --git a/src/toil/test/src/dockerCheckTest.py b/src/toil/test/src/dockerCheckTest.py index 0a1c857a5f..999a8302ef 100644 --- a/src/toil/test/src/dockerCheckTest.py +++ b/src/toil/test/src/dockerCheckTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,21 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from docker.errors import ImageNotFound from toil import checkDockerImageExists, parseDockerAppliance -from toil.test import ToilTest, needs_appliance - +from toil.test import ToilTest, needs_docker -# requires internet -@needs_appliance -class dockerCheckTests(ToilTest): - """ - Tests initial checking of whether a docker image exists in the specified repository or not. - """ - def setUp(self): - pass +@needs_docker +class DockerCheckTest(ToilTest): + """Tests checking whether a docker image exists or not.""" def testOfficialUbuntuRepo(self): """Image exists. This should pass.""" ubuntu_repo = 'ubuntu:latest' diff --git a/src/toil/test/src/fileStoreTest.py b/src/toil/test/src/fileStoreTest.py index 27e085607c..12586ea0d9 100644 --- a/src/toil/test/src/fileStoreTest.py +++ b/src/toil/test/src/fileStoreTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - import collections import datetime import errno @@ -45,7 +43,7 @@ logger = logging.getLogger(__name__) -class hidden(object): +class hidden: """ Hiding the abstract test classes from the Unittest loader so it can be inherited in different test suites for the different job stores. @@ -108,7 +106,7 @@ def testFileStoreLogging(self): Write a couple of files to the jobstore. Delete a couple of them. Read back written and locally deleted files. """ - + class WatchingHandler(logging.Handler): """ A logging handler that watches for a certain substring and @@ -121,11 +119,11 @@ def __init__(self, match: str): def emit(self, record): if self.match in record.getMessage(): self.seen = True - + handler = WatchingHandler("cats.txt") - + logging.getLogger().addHandler(handler) - + F = Job.wrapJobFn(self._accessAndFail, disk='100M') try: @@ -133,11 +131,11 @@ def emit(self, record): except FailedJobsException: # We expect this. pass - + logging.getLogger().removeHandler(handler) - + assert handler.seen, "Downloaded file name not found in logs of failing Toil run" - + @staticmethod def _accessAndFail(job): with job.fileStore.writeGlobalFileStream() as (stream, fileID): @@ -147,7 +145,7 @@ def _accessAndFail(job): with job.fileStore.readGlobalFileStream(fileID) as stream2: pass raise RuntimeError("I do not like this file") - + # Test filestore operations. This is a slightly less intense version of the cache specific # test `testReturnFileSizes` @@ -182,7 +180,7 @@ def _testFileStoreOperations(job, nonLocalDir, numIters=100): # Remember it actually should be local localFileIDs.add(fsId) logger.info('Now have local file: %s', fsId) - + i = 0 while i <= numIters: randVal = random.random() @@ -412,7 +410,7 @@ def _writeFileToJobStoreWithAsserts(job, isLocalFile, nonLocalDir=None, fileMB=1 is created. :param int fileMB: Size of the created file in MB :param bool expectAsyncUpload: Whether we expect the upload to hit - the job store later(T) or immediately(F) + the job store later(T) or immediately(F) """ cls = hidden.AbstractNonCachingFileStoreTest fsID, testFile = cls._writeFileToJobStore(job, isLocalFile, nonLocalDir, fileMB) @@ -440,7 +438,7 @@ def _writeFileToJobStoreWithAsserts(job, isLocalFile, nonLocalDir=None, fileMB=1 if not isLocalFile: # Make sure it isn't cached if we don't want it to be assert not job.fileStore.fileIsCached(fsID), "File uploaded from non-local-temp directory %s should not be cached" % nonLocalDir - + return fsID @staticmethod @@ -459,7 +457,7 @@ def _adjustCacheLimit(job, newTotalMB): newTotalMB, changing the maximum cache disk space allowed for the run. - :param int newTotalMB: New total cache disk space limit in MB. + :param int newTotalMB: New total cache disk space limit in MB. """ # Convert to bytes and pass on to the actual cache @@ -548,7 +546,7 @@ def _doubleWriteFileToJobStore(job, fileMB): job.fileStore.logToMaster('Copy 2 ID: {}'.format(fsID)) hidden.AbstractCachingFileStoreTest._readFromJobStoreWithoutAssertions(job, fsID) - + job.fileStore.logToMaster('Writing copy 3 and returning ID') return job.fileStore.writeGlobalFile(testFile.name) @@ -565,7 +563,7 @@ def _readFromJobStoreWithoutAssertions(job, fsID): job.fileStore.readGlobalFile(fsID) # writeGlobalFile tests - + @travis_test def testWriteNonLocalFileToJobStore(self): """ @@ -576,7 +574,7 @@ def testWriteNonLocalFileToJobStore(self): A = Job.wrapJobFn(self._writeFileToJobStoreWithAsserts, isLocalFile=False, nonLocalDir=workdir) Job.Runner.startToil(A, self.options) - + @travis_test def testWriteLocalFileToJobStore(self): """ @@ -587,7 +585,7 @@ def testWriteLocalFileToJobStore(self): Job.Runner.startToil(A, self.options) # readGlobalFile tests - + @travis_test def testReadCacheMissFileFromJobStoreWithoutCachingReadFile(self): """ @@ -595,7 +593,7 @@ def testReadCacheMissFileFromJobStoreWithoutCachingReadFile(self): cache the read file. Ensure the number of links on the file are appropriate. """ self._testCacheMissFunction(cacheReadFile=False) - + @travis_test def testReadCacheMissFileFromJobStoreWithCachingReadFile(self): """ @@ -662,7 +660,7 @@ def _readFromJobStore(job, isCachedFile, cacheReadFile, fsID, isTest=True): return None else: return outfile - + @travis_test def testReadCachHitFileFromJobStore(self): """ @@ -814,7 +812,7 @@ def testReturnFileSizesWithBadWorker(self): """ Write a couple of files to the jobstore. Delete a couple of them. Read back written and locally deleted files. Ensure that after - every step that the cache is in a valid state. + every step that the cache is in a valid state. """ self.options.retryCount = 20 self.options.badWorker = 0.5 @@ -831,10 +829,10 @@ def testReturnFileSizesWithBadWorker(self): def _returnFileTestFn(job, jobDisk, initialCachedSize, nonLocalDir, numIters=100): """ Aux function for jobCacheTest.testReturnFileSizes Conduct numIters operations and ensure - the cache has the right amount of data in it at all times. + the cache has the right amount of data in it at all times. Track the cache calculations even thought they won't be used in filejobstore - + Assumes nothing is evicted from the cache. :param float jobDisk: The value of disk passed to this job. @@ -844,7 +842,7 @@ def _returnFileTestFn(job, jobDisk, initialCachedSize, nonLocalDir, numIters=100 work_dir = job.fileStore.getLocalTempDir() writtenFiles = {} # fsID: (size, isLocal) # fsid: local/mutable/immutable for all operations that should make local files as tracked by the FileStore - localFileIDs = collections.defaultdict(list) + localFileIDs = collections.defaultdict(list) # Add one file for the sake of having something in the job store writeFileSize = random.randint(0, 30) jobDisk -= writeFileSize * 1024 * 1024 @@ -973,9 +971,9 @@ def _requirementsConcur(job, jobDisk, cached): else: RealtimeLogger.info('Caching is free; %d bytes are used and %d bytes would be expected if caching were not free', used, cached) assert used == 0, 'Cache should have nothing in it, but actually has %d bytes used' % used - + jobUnused = job.fileStore.getCacheUnusedJobRequirement() - + assert jobUnused == jobDisk, 'Job should have %d bytes of disk for non-FileStore use but the FileStore reports %d' % (jobDisk, jobUnused) # Testing the resumability of a failed worker @@ -1000,7 +998,7 @@ def _controlledFailTestFn(job, jobDisk, testDir): """ This is the aux function for the controlled failed worker test. It does a couple of cache operations, fails, then checks whether the new worker starts with the expected - value, and whether it computes cache statistics correctly. + value, and whether it computes cache statistics correctly. :param float jobDisk: Disk space supplied for this job :param str testDir: Testing directory @@ -1088,7 +1086,7 @@ def _removeReadFileFn(job, fileToDelete, readAsMutable): outfile = testFile.name else: break - + @travis_test def testDeleteLocalFile(self): """ @@ -1164,7 +1162,7 @@ def testSimultaneousReadsUncachedStream(self): """ self.options.retryCount = 0 self.options.disableChaining = True - + # Make a file parent = Job.wrapJobFn(self._createUncachedFileStream) # Now make a bunch of children fight over it @@ -1206,7 +1204,7 @@ def _readFileWithDelay(job, fileID, cores=0.1, memory=50 * 1024 * 1024, disk=50 readStart = datetime.datetime.now() logger.debug('Begin read at %s', str(readStart)) - + localPath = job.fileStore.readGlobalFile(fileID, cache=True, mutable=True) readEnd = datetime.datetime.now() @@ -1215,7 +1213,7 @@ def _readFileWithDelay(job, fileID, cores=0.1, memory=50 * 1024 * 1024, disk=50 with open(localPath, 'rb') as fh: text = fh.read().decode('utf-8').strip() logger.debug('Got file contents: %s', text) - + class NonCachingFileStoreTestWithFileJobStore(hidden.AbstractNonCachingFileStoreTest): diff --git a/src/toil/test/src/helloWorldTest.py b/src/toil/test/src/helloWorldTest.py index 9e6bf69eca..04d04f7df4 100644 --- a/src/toil/test/src/helloWorldTest.py +++ b/src/toil/test/src/helloWorldTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/src/importExportFileTest.py b/src/toil/test/src/importExportFileTest.py index f865ce04d5..062da89517 100644 --- a/src/toil/test/src/importExportFileTest.py +++ b/src/toil/test/src/importExportFileTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -111,4 +111,3 @@ def run(self, fileStore): with fileStore.writeGlobalFileStream() as (fo, outputFileID): fo.write((fi.read().decode('utf-8') + 'World!').encode('utf-8')) return outputFileID - diff --git a/src/toil/test/src/jobDescriptionTest.py b/src/toil/test/src/jobDescriptionTest.py index 1c8d1c3cd9..9638cf31cb 100644 --- a/src/toil/test/src/jobDescriptionTest.py +++ b/src/toil/test/src/jobDescriptionTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -97,4 +97,3 @@ def testJobDescriptionSequencing(self): # empty list. Nothing left to do! j.filterSuccessors(lambda jID: jID != 'followOn') self.assertEqual(j.nextSuccessors(), None) - diff --git a/src/toil/test/src/jobEncapsulationTest.py b/src/toil/test/src/jobEncapsulationTest.py index f8f2888a8a..5a7d79b288 100644 --- a/src/toil/test/src/jobEncapsulationTest.py +++ b/src/toil/test/src/jobEncapsulationTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ import os from toil.job import Job -from toil.lib.bioio import getTempFile +from toil.test import get_temp_file from toil.test import ToilTest, travis_test from toil.test.src.jobTest import fn1Test @@ -31,7 +31,7 @@ def testEncapsulation(self): class. """ # Temporary file - outFile = getTempFile(rootDir=self._createTempDir()) + outFile = get_temp_file(rootDir=self._createTempDir()) try: # Encapsulate a job graph a = Job.wrapJobFn(encapsulatedJobFn, "A", outFile, name="a") diff --git a/src/toil/test/src/jobFileStoreTest.py b/src/toil/test/src/jobFileStoreTest.py index eb61e3e792..6c82c726f1 100644 --- a/src/toil/test/src/jobFileStoreTest.py +++ b/src/toil/test/src/jobFileStoreTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/src/jobServiceTest.py b/src/toil/test/src/jobServiceTest.py index eef383ecab..ba60cd88ba 100644 --- a/src/toil/test/src/jobServiceTest.py +++ b/src/toil/test/src/jobServiceTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,16 +19,15 @@ import time import traceback from threading import Event, Thread - -from toil.job import Job -from toil.lib.bioio import getTempFile -from toil.test import ToilTest, slow - -logger = logging.getLogger( __name__ ) from unittest import skipIf from toil.batchSystems.singleMachine import SingleMachineBatchSystem +from toil.job import Job from toil.leader import DeadlockException, FailedJobsException +from toil.test import get_temp_file +from toil.test import ToilTest, slow + +logger = logging.getLogger(__name__) class JobServiceTest(ToilTest): @@ -58,7 +57,7 @@ def testService(self, checkpoint=False): Tests the creation of a Job.Service with random failures of the worker. """ for test in range(2): - outFile = getTempFile(rootDir=self._createTempDir()) # Temporary file + outFile = get_temp_file(rootDir=self._createTempDir()) # Temporary file messageInt = random.randint(1, sys.maxsize) try: # Wire up the services/jobs @@ -78,7 +77,7 @@ def testServiceDeadlock(self): """ Creates a job with more services than maxServices, checks that deadlock is detected. """ - outFile = getTempFile(rootDir=self._createTempDir()) + outFile = get_temp_file(rootDir=self._createTempDir()) try: def makeWorkflow(): job = Job() @@ -119,7 +118,7 @@ def testServiceRecursive(self, checkpoint=True): """ for test in range(1): # Temporary file - outFile = getTempFile(rootDir=self._createTempDir()) + outFile = get_temp_file(rootDir=self._createTempDir()) messages = [ random.randint(1, sys.maxsize) for i in range(3) ] try: # Wire up the services/jobs @@ -142,7 +141,7 @@ def testServiceParallelRecursive(self, checkpoint=True): """ for test in range(1): # Temporary file - outFiles = [ getTempFile(rootDir=self._createTempDir()) for j in range(2) ] + outFiles = [get_temp_file(rootDir=self._createTempDir()) for j in range(2)] messageBundles = [ [ random.randint(1, sys.maxsize) for i in range(3) ] for j in range(2) ] try: # Wire up the services/jobs @@ -369,5 +368,3 @@ def fnTest(strings, outputFile): """ with open(outputFile, 'w') as fH: fH.write(" ".join(strings)) - - diff --git a/src/toil/test/src/jobTest.py b/src/toil/test/src/jobTest.py index dbcee7bca9..4e789c6e80 100644 --- a/src/toil/test/src/jobTest.py +++ b/src/toil/test/src/jobTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ from toil.common import Toil from toil.job import Job, JobFunctionWrappingJob, JobGraphDeadlockException from toil.leader import FailedJobsException -from toil.lib.bioio import getTempFile +from toil.test import get_temp_file from toil.test import ToilTest, slow, travis_test logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def testStatic(self): Follow on is marked by -> """ - outFile = getTempFile(rootDir=self._createTempDir()) + outFile = get_temp_file(rootDir=self._createTempDir()) try: # Create the jobs @@ -95,7 +95,7 @@ def testStatic2(self): Follow on is marked by -> """ - outFile = getTempFile(rootDir=self._createTempDir()) + outFile = get_temp_file(rootDir=self._createTempDir()) try: # Create the jobs diff --git a/src/toil/test/src/miscTests.py b/src/toil/test/src/miscTests.py index c241649b0f..1ab5da8ece 100644 --- a/src/toil/test/src/miscTests.py +++ b/src/toil/test/src/miscTests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,8 +21,11 @@ from toil.common import getNodeID from toil.lib.exceptions import panic, raise_ -from toil.lib.misc import (AtomicFileCreate, CalledProcessErrorStderr, - atomic_install, atomic_tmp_file, call_command) +from toil.lib.misc import (AtomicFileCreate, + CalledProcessErrorStderr, + atomic_install, + atomic_tmp_file, + call_command) from toil.test import ToilTest, slow, travis_test log = logging.getLogger(__name__) diff --git a/src/toil/test/src/promisedRequirementTest.py b/src/toil/test/src/promisedRequirementTest.py index 8b202710d9..1a2b464345 100644 --- a/src/toil/test/src/promisedRequirementTest.py +++ b/src/toil/test/src/promisedRequirementTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -235,4 +235,3 @@ def getBatchSystemName(self): def tearDown(self): self._stopMesos() - diff --git a/src/toil/test/src/promisesTest.py b/src/toil/test/src/promisesTest.py index 83f4508775..b22da33629 100644 --- a/src/toil/test/src/promisesTest.py +++ b/src/toil/test/src/promisesTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/src/realtimeLoggerTest.py b/src/toil/test/src/realtimeLoggerTest.py index eb0f6978e2..7db25e7077 100644 --- a/src/toil/test/src/realtimeLoggerTest.py +++ b/src/toil/test/src/realtimeLoggerTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/src/regularLogTest.py b/src/toil/test/src/regularLogTest.py index a07f22d1f8..e9ec87440a 100644 --- a/src/toil/test/src/regularLogTest.py +++ b/src/toil/test/src/regularLogTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) + class RegularLogTest(ToilTest): def setUp(self): @@ -60,7 +61,7 @@ def testLogToMaster(self): '--clean=always', '--logLevel=info'], stderr=subprocess.STDOUT) assert helloWorld.childMessage in toilOutput.decode('utf-8') - + @travis_test def testWriteLogs(self): subprocess.check_call([sys.executable, diff --git a/src/toil/test/src/resourceTest.py b/src/toil/test/src/resourceTest.py index 435e4c5316..47d69aeadf 100644 --- a/src/toil/test/src/resourceTest.py +++ b/src/toil/test/src/resourceTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,12 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - import importlib import os import subprocess import sys +import tempfile +from contextlib import contextmanager from inspect import getsource from io import BytesIO from textwrap import dedent @@ -26,10 +26,31 @@ from toil import inVirtualEnv from toil.resource import ModuleDescriptor, Resource, ResourceException -from toil.test import ToilTest, tempFileContaining, travis_test +from toil.test import ToilTest, travis_test from toil.version import exactPython +@contextmanager +def tempFileContaining(content, suffix=''): + """ + Write a file with the given contents, and keep it on disk as long as the context is active. + :param str content: The contents of the file. + :param str suffix: The extension to use for the temporary file. + """ + fd, path = tempfile.mkstemp(suffix=suffix) + try: + encoded = content.encode('utf-8') + assert os.write(fd, encoded) == len(encoded) + except: + os.close(fd) + raise + else: + os.close(fd) + yield path + finally: + os.unlink(path) + + class ResourceTest(ToilTest): """ Test module descriptors and resources derived from them. diff --git a/src/toil/test/src/restartDAGTest.py b/src/toil/test/src/restartDAGTest.py index 8d57ea5713..053a450bb7 100644 --- a/src/toil/test/src/restartDAGTest.py +++ b/src/toil/test/src/restartDAGTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/src/resumabilityTest.py b/src/toil/test/src/resumabilityTest.py index f8f948c9e2..a82c71d940 100644 --- a/src/toil/test/src/resumabilityTest.py +++ b/src/toil/test/src/resumabilityTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/src/retainTempDirTest.py b/src/toil/test/src/retainTempDirTest.py index 00d6955f70..6fe96d4f0c 100644 --- a/src/toil/test/src/retainTempDirTest.py +++ b/src/toil/test/src/retainTempDirTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/src/threadingTest.py b/src/toil/test/src/threadingTest.py index c9487c7fa9..6300bd0b85 100644 --- a/src/toil/test/src/threadingTest.py +++ b/src/toil/test/src/threadingTest.py @@ -135,4 +135,3 @@ def _testLastProcessStandingTask(scope, arena_name, number): except: traceback.print_exc() return False - diff --git a/src/toil/test/src/toilContextManagerTest.py b/src/toil/test/src/toilContextManagerTest.py index d4d06d727c..bcfcba3375 100644 --- a/src/toil/test/src/toilContextManagerTest.py +++ b/src/toil/test/src/toilContextManagerTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,14 +16,14 @@ from toil.common import Toil, ToilContextManagerException from toil.job import Job -from toil.lib.bioio import getTempFile +from toil.test import get_temp_file from toil.test import ToilTest, slow @slow class ToilContextManagerTest(ToilTest): def setUp(self): - self.exportPath = getTempFile() + self.exportPath = get_temp_file() def tearDown(self): os.remove(self.exportPath) diff --git a/src/toil/test/src/userDefinedJobArgTypeTest.py b/src/toil/test/src/userDefinedJobArgTypeTest.py index 5bd7443108..ff7a1def2f 100644 --- a/src/toil/test/src/userDefinedJobArgTypeTest.py +++ b/src/toil/test/src/userDefinedJobArgTypeTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/src/workerTest.py b/src/toil/test/src/workerTest.py index 5fbd405a59..9460577f5e 100644 --- a/src/toil/test/src/workerTest.py +++ b/src/toil/test/src/workerTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/utils/__init__.py b/src/toil/test/utils/__init__.py index f7d9550f46..471fe7cd32 100644 --- a/src/toil/test/utils/__init__.py +++ b/src/toil/test/utils/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/utils/toilDebugTest.py b/src/toil/test/utils/toilDebugTest.py index 2ba1f621ef..d3a49e0a4d 100644 --- a/src/toil/test/utils/toilDebugTest.py +++ b/src/toil/test/utils/toilDebugTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/utils/utilsTest.py b/src/toil/test/utils/utilsTest.py index 765784409b..1b19868611 100644 --- a/src/toil/test/utils/utilsTest.py +++ b/src/toil/test/utils/utilsTest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,29 +14,32 @@ import logging import os import shutil +import subprocess import sys import tempfile +import time import uuid import pytest +from mock import patch pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) # noqa sys.path.insert(0, pkg_root) # noqa -import subprocess -import time - -from mock import patch - import toil -import toil.test.sort.sort from toil import resolveEntryPoint from toil.common import Config, Toil from toil.job import Job -from toil.lib.bioio import getTempFile, system -from toil.provisioners import clusterFactory -from toil.test import (ToilTest, integrative, needs_aws_ec2, needs_cwl, - needs_docker, needs_rsync3, slow, travis_test) +from toil.lib.bioio import system +from toil.test import (ToilTest, + integrative, + needs_aws_ec2, + needs_cwl, + needs_docker, + needs_rsync3, + slow, + travis_test, + get_temp_file) from toil.test.sort.sortTest import makeFileToSort from toil.utils.toilStats import getStats, processData from toil.utils.toilStatus import ToilStatus @@ -54,7 +57,7 @@ class UtilsTest(ToilTest): def setUp(self): super(UtilsTest, self).setUp() self.tempDir = self._createTempDir() - self.tempFile = getTempFile(rootDir=self.tempDir) + self.tempFile = get_temp_file(rootDir=self.tempDir) self.outputFile = 'someSortedStuff.txt' self.toilDir = os.path.join(self.tempDir, "jobstore") self.assertFalse(os.path.exists(self.toilDir)) @@ -67,13 +70,22 @@ def setUp(self): self.correctSort = fileHandle.readlines() self.correctSort.sort() - self.sort_workflow_cmd = [python, '-m', 'toil.test.sort.sort', - 'file:' + self.toilDir, - '--clean=never', - '--numLines=1', '--lineLength=1'] - - self.restart_sort_workflow_cmd = [python, '-m', 'toil.test.sort.restart_sort', - 'file:' + self.toilDir] + self.sort_workflow_cmd = [ + python, + '-m', + 'toil.test.sort.sort', + f'file:{self.toilDir}', + '--clean=never', + '--numLines=1', + '--lineLength=1' + ] + + self.restart_sort_workflow_cmd = [ + python, + '-m', + 'toil.test.sort.restart_sort', + f'file:{self.toilDir}' + ] def tearDown(self): if os.path.exists(self.tempDir): @@ -125,90 +137,29 @@ def testAWSProvisionerUtils(self): :return: """ # TODO: Run these for the other clouds. - clusterName = 'cluster-utils-test' + str(uuid.uuid4()) - keyName = os.getenv('TOIL_AWS_KEYNAME') + clusterName = f'cluster-utils-test{uuid.uuid4()}' + keyName = os.getenv('TOIL_AWS_KEYNAME').strip() or 'id_rsa' try: from toil.provisioners.aws.awsProvisioner import AWSProvisioner - logger.debug("Found AWSProvisioner: %s.", AWSProvisioner.__file__) + aws_provisioner = AWSProvisioner.__module__ + logger.debug(f"Found AWSProvisioner: {aws_provisioner}.") # launch master with an assortment of custom tags system([self.toilMain, 'launch-cluster', '-t', 'key1=value1', '-t', 'key2=value2', '--tag', 'key3=value3', '--leaderNodeType=m3.medium', '--keyPairName=' + keyName, clusterName, '--provisioner=aws', '--zone=us-west-2a', '--logLevel=DEBUG']) - cluster = clusterFactory(provisioner='aws', clusterName=clusterName) + from toil.provisioners import cluster_factory + cluster = toil.provisioners.cluster_factory(provisioner='aws', clusterName=clusterName) leader = cluster.getLeader() # check that the leader carries the appropriate tags tags = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3', 'Name': clusterName, 'Owner': keyName} for key in tags: self.assertEqual(tags[key], leader.tags.get(key)) - - # Test strict host key checking - # Doesn't work when run locally. - if keyName == 'jenkins@jenkins-master': - try: - leader.sshAppliance(strict=True) - except RuntimeError: - pass - else: - self.fail("Host key verification passed where it should have failed") - - # Add the host key to known_hosts so that the rest of the tests can - # pass without choking on the verification prompt. - leader.sshAppliance('bash', strict=True, sshOptions=['-oStrictHostKeyChecking=no']) - - system([self.toilMain, 'ssh-cluster', '--provisioner=aws', clusterName]) - - testStrings = ["'foo'", - '"foo"', - ' foo', - '$PATH', - '"', - "'", - '\\', - '| cat', - '&& cat', - '; cat'] - for test in testStrings: - logger.debug('Testing SSH with special string: %s', test) - compareTo = "import sys; assert sys.argv[1]==%r" % test - leader.sshAppliance(python, '-', test, input=compareTo) - - try: - leader.sshAppliance('nonsenseShouldFail') - except RuntimeError: - pass - else: - self.fail('The remote command failed silently where it should have raised an error') - - leader.sshAppliance(python, '-c', "import os; assert os.environ['TOIL_WORKDIR']=='/var/lib/toil'") - - # `toil rsync-cluster` - # Testing special characters - string.punctuation - fname = r'!"#$%&\'()*+,-.;<=>:\ ?@[\\]^_`{|}~' - testData = os.urandom(3 * (10**6)) - with tempfile.NamedTemporaryFile(suffix=fname) as tmpFile: - relpath = os.path.basename(tmpFile.name) - tmpFile.write(testData) - tmpFile.flush() - # Upload file to leader - leader.coreRsync(args=[tmpFile.name, ":"]) - # Ensure file exists - leader.sshAppliance("test", "-e", relpath) - tmpDir = tempfile.mkdtemp() - # Download the file again and make sure it's the same file - # `--protect-args` needed because remote bash chokes on special characters - leader.coreRsync(args=["--protect-args", ":" + relpath, tmpDir]) - with open(os.path.join(tmpDir, relpath), "r") as f: - self.assertEqual(f.read(), testData, "Downloaded file does not match original file") finally: system([self.toilMain, 'destroy-cluster', '--provisioner=aws', clusterName]) - try: - shutil.rmtree(tmpDir) - except NameError: - pass @slow def testUtilsSort(self): diff --git a/src/toil/test/wdl/builtinTest.py b/src/toil/test/wdl/builtinTest.py index e4aa7004f2..b381c1704d 100644 --- a/src/toil/test/wdl/builtinTest.py +++ b/src/toil/test/wdl/builtinTest.py @@ -8,28 +8,27 @@ from toil.test import ToilTest from toil.version import exactPython -from toil.wdl.wdl_functions import ( - ceil, - floor, - length, - read_boolean, - read_float, - read_int, - read_json, - read_lines, - read_map, - read_string, - read_tsv, - sub, - transpose, - write_json, - write_lines, - write_map, - write_tsv, - wdl_zip, - cross, - WDLPair, - WDLRuntimeError) +from toil.wdl.wdl_functions import (WDLPair, + WDLRuntimeError, + ceil, + cross, + floor, + length, + read_boolean, + read_float, + read_int, + read_json, + read_lines, + read_map, + read_string, + read_tsv, + sub, + transpose, + wdl_zip, + write_json, + write_lines, + write_map, + write_tsv) class WdlStandardLibraryFunctionsTest(ToilTest): diff --git a/src/toil/test/wdl/conftest.py b/src/toil/test/wdl/conftest.py index bc2be26cc9..5e5664f827 100644 --- a/src/toil/test/wdl/conftest.py +++ b/src/toil/test/wdl/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/toil/test/wdl/toilwdlTest.py b/src/toil/test/wdl/toilwdlTest.py index 5a2c81c777..057841a8ec 100644 --- a/src/toil/test/wdl/toilwdlTest.py +++ b/src/toil/test/wdl/toilwdlTest.py @@ -7,30 +7,26 @@ from urllib.request import urlretrieve import toil.wdl.wdl_parser as wdl_parser -from toil.test import ToilTest, needs_docker, slow +from toil.test import ToilTest, needs_docker, needs_java, slow from toil.version import exactPython from toil.wdl.wdl_analysis import AnalyzeWDL +from toil.wdl.wdl_functions import (abspath_file, + basename, + combine_dicts, + defined, + generate_docker_bashscript_file, + glob, + parse_cores, + parse_disk, + parse_memory, + process_and_read_file, + process_infile, + process_outfile, + read_csv, + read_tsv, + select_first, + size) from toil.wdl.wdl_synthesis import SynthesizeWDL -from toil.wdl.wdl_functions import generate_docker_bashscript_file -from toil.wdl.wdl_functions import select_first -from toil.wdl.wdl_functions import size -from toil.wdl.wdl_functions import glob -from toil.wdl.wdl_functions import process_and_read_file -from toil.wdl.wdl_functions import process_infile -from toil.wdl.wdl_functions import process_outfile -from toil.wdl.wdl_functions import abspath_file -from toil.wdl.wdl_functions import combine_dicts -from toil.wdl.wdl_functions import parse_memory -from toil.wdl.wdl_functions import parse_cores -from toil.wdl.wdl_functions import parse_disk -from toil.wdl.wdl_functions import defined -from toil.wdl.wdl_functions import read_tsv -from toil.wdl.wdl_functions import read_csv -from toil.wdl.wdl_functions import basename -from toil.test import ToilTest, slow, needs_docker, needs_java -import zipfile -import shutil -import uuid class ToilWdlIntegrationTest(ToilTest): diff --git a/src/toil/toilState.py b/src/toil/toilState.py index 54036e4c08..57e4ba16bd 100644 --- a/src/toil/toilState.py +++ b/src/toil/toilState.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging from toil.job import CheckpointJobDescription, JobDescription @@ -19,7 +18,7 @@ logger = logging.getLogger(__name__) -class ToilState(): +class ToilState: """ Holds the leader's scheduling information that does not need to be persisted back to the JobStore (such as information on completed and diff --git a/src/toil/utils/__init__.py b/src/toil/utils/__init__.py index b819abe9ba..e69de29bb2 100644 --- a/src/toil/utils/__init__.py +++ b/src/toil/utils/__init__.py @@ -1,36 +0,0 @@ - -import logging -import os - -from toil import version - -logger = logging.getLogger(__name__) - - -def addBasicProvisionerOptions(parser): - parser.add_argument("--version", action='version', version=version) - parser.add_argument('-p', "--provisioner", dest='provisioner', choices=['aws', 'gce'], required=False, default="aws", - help="The provisioner for cluster auto-scaling. AWS and Google are currently supported") - parser.add_argument('-z', '--zone', dest='zone', required=False, default=None, - help="The availability zone of the master. This parameter can also be set via the 'TOIL_X_ZONE' " - "environment variable, where X is AWS or GCE, or by the ec2_region_name parameter " - "in your .boto file, or derived from the instance metadata if using this utility on an " - "existing EC2 instance.") - parser.add_argument("clusterName", help="The name that the cluster will be identifiable by. " - "Must be lowercase and may not contain the '_' " - "character.") - return parser - - -def getZoneFromEnv(provisioner): - """ - Find the zone specified in an environment variable. - - The user can specify zones in environment variables in lieu of writing them at the commandline every time. - Given a provisioner, this method will look for the stored value and return it. - :param str provisioner: One of the supported provisioners ('aws', 'gce') - :rtype: str - :return: None or the value stored in a 'TOIL_X_ZONE' environment variable. - """ - - return os.environ.get('TOIL_' + provisioner.upper() + '_ZONE') diff --git a/src/toil/utils/toilClean.py b/src/toil/utils/toilClean.py index c5c2e83f00..0df190f578 100644 --- a/src/toil/utils/toilClean.py +++ b/src/toil/utils/toilClean.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,32 +11,28 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Delete the job store used by a previous Toil workflow invocation -""" +"""Delete a job store used by a previous Toil workflow invocation.""" import logging -from toil.common import Config, Toil, jobStoreLocatorHelp +from toil.common import Toil, parser_with_common_options from toil.jobStores.abstractJobStore import NoSuchJobStoreException -from toil.lib.bioio import getBasicOptionParser, parseBasicOptions -from toil.version import version +from toil.statsAndLogging import set_logging_from_options + +logger = logging.getLogger(__name__) -logger = logging.getLogger( __name__ ) def main(): - parser = getBasicOptionParser() - parser.add_argument("jobStore", type=str, - help="The location of the job store to delete. " + jobStoreLocatorHelp) - parser.add_argument("--version", action='version', version=version) - config = Config() - config.setOptions(parseBasicOptions(parser)) + parser = parser_with_common_options(jobstore_option=True) + + options = parser.parse_args() + set_logging_from_options(options) try: - jobStore = Toil.getJobStore(config.jobStore) - jobStore.resume() - jobStore.destroy() - logger.info("Successfully deleted the job store: %s" % config.jobStore) + jobstore = Toil.getJobStore(options.jobStore) + jobstore.resume() + jobstore.destroy() + logger.info(f"Successfully deleted the job store: {options.jobStore}") except NoSuchJobStoreException: - logger.info("Failed to delete the job store: %s is non-existent" % config.jobStore) + logger.info(f"Failed to delete the job store: {options.jobStore} is non-existent.") except: - logger.info("Failed to delete the job store: %s" % config.jobStore) + logger.info(f"Failed to delete the job store: {options.jobStore}") raise diff --git a/src/toil/utils/toilDebugFile.py b/src/toil/utils/toilDebugFile.py index d0de5b0aea..edf622232d 100644 --- a/src/toil/utils/toilDebugFile.py +++ b/src/toil/utils/toilDebugFile.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,19 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -"""Debug tool for copying files contained in a toil jobStore. -""" - +"""Debug tool for copying files contained in a toil jobStore.""" import fnmatch import logging import os.path -from toil.common import Config, Toil, jobStoreLocatorHelp -from toil.lib.bioio import getBasicOptionParser, parseBasicOptions -from toil.version import version +from toil.common import Config, Toil, parser_with_common_options +from toil.statsAndLogging import set_logging_from_options + +logger = logging.getLogger(__name__) -logger = logging.getLogger( __name__ ) def recursiveGlob(directoryname, glob_pattern): ''' @@ -42,6 +39,7 @@ def recursiveGlob(directoryname, glob_pattern): matches.append(absolute_filepath) return matches + def fetchJobStoreFiles(jobStore, options): """ Takes a list of file names as glob patterns, searches for these within a @@ -58,14 +56,13 @@ def fetchJobStoreFiles(jobStore, options): jobStoreHits = recursiveGlob(directoryname=options.jobStore, glob_pattern=jobStoreFile) for jobStoreFileID in jobStoreHits: - logger.debug("Copying job store file: %s to %s", - jobStoreFileID, - options.localFilePath[0]) + logger.debug(f"Copying job store file: {jobStoreFileID} to {options.localFilePath[0]}") jobStore.readFile(jobStoreFileID, os.path.join(options.localFilePath[0], - os.path.basename(jobStoreFileID)), + os.path.basename(jobStoreFileID)), symlink=options.useSymlinks) + def printContentsOfJobStore(jobStorePath, nameOfJob=None): """ Fetch a list of all files contained in the jobStore directory input if @@ -92,38 +89,34 @@ def printContentsOfJobStore(jobStorePath, nameOfJob=None): os.remove(logFile) for gfile in sorted(list_of_files): if not gfile.endswith('.new'): - logger.debug(nameOfJob + "File: %s", os.path.basename(gfile)) + logger.debug(f"{nameOfJob} File: {os.path.basename(gfile)}") with open(logFile, "a+") as f: - f.write(os.path.basename(gfile)) - f.write("\n") + f.write(os.path.basename(gfile)) + f.write("\n") -def main(): - parser = getBasicOptionParser() - parser.add_argument("jobStore", - type=str, - help="The location of the job store used by the workflow." + - jobStoreLocatorHelp) +def main(): + parser = parser_with_common_options(jobstore_option=True) parser.add_argument("--localFilePath", nargs=1, help="Location to which to copy job store files.") parser.add_argument("--fetch", nargs="+", help="List of job-store files to be copied locally." - "Use either explicit names (i.e. 'data.txt'), or " - "specify glob patterns (i.e. '*.txt')") + "Use either explicit names (i.e. 'data.txt'), or " + "specify glob patterns (i.e. '*.txt')") parser.add_argument("--listFilesInJobStore", help="Prints a list of the current files in the jobStore.") parser.add_argument("--fetchEntireJobStore", help="Copy all job store files into a local directory.") parser.add_argument("--useSymlinks", help="Creates symlink 'shortcuts' of files in the localFilePath" - " instead of hardlinking or copying, where possible. If this is" - " not possible, it will copy the files (shutil.copyfile()).") - parser.add_argument("--version", action='version', version=version) - + " instead of hardlinking or copying, where possible. If this is" + " not possible, it will copy the files (shutil.copyfile()).") + # Load the jobStore - options = parseBasicOptions(parser) + options = parser.parse_args() + set_logging_from_options(options) config = Config() config.setOptions(options) jobStore = Toil.resumeJobStore(config.jobStore) @@ -144,5 +137,6 @@ def main(): # Log filenames and create a file containing these names in cwd printContentsOfJobStore(jobStorePath=options.jobStore) -if __name__=="__main__": + +if __name__ == "__main__": main() diff --git a/src/toil/utils/toilDebugJob.py b/src/toil/utils/toilDebugJob.py index 644b4a0bdd..c42c36fe51 100644 --- a/src/toil/utils/toilDebugJob.py +++ b/src/toil/utils/toilDebugJob.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017- Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,53 +11,40 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -"""Debug tool for running a toil job locally. -""" - +"""Debug tool for running a toil job locally.""" import logging -from toil.common import Config, Toil, jobStoreLocatorHelp -from toil.lib.bioio import getBasicOptionParser, parseBasicOptions +from toil.common import Config, Toil, parser_with_common_options +from toil.statsAndLogging import set_logging_from_options from toil.utils.toilDebugFile import printContentsOfJobStore -from toil.version import version from toil.worker import workerScript -logger = logging.getLogger( __name__ ) +logger = logging.getLogger(__name__) -def print_successor_jobs(): - pass def main(): - parser = getBasicOptionParser() - - parser.add_argument("jobStore", type=str, - help="The location of the job store used by the workflow." + jobStoreLocatorHelp) - parser.add_argument("jobID", nargs=1, help="The job store id of a job " - "within the provided jobstore to run by itself.") + parser = parser_with_common_options(jobstore_option=True) + parser.add_argument("jobID", nargs=1, + help="The job store id of a job within the provided jobstore to run by itself.") parser.add_argument("--printJobInfo", nargs=1, - help="Return information about this job to the user" - " including preceding jobs, inputs, outputs, and runtime" - " from the last known run.") - parser.add_argument("--version", action='version', version=version) - - # Parse options - options = parseBasicOptions(parser) + help="Return information about this job to the user including preceding jobs, " + "inputs, outputs, and runtime from the last known run.") + + options = parser.parse_args() + set_logging_from_options(options) config = Config() config.setOptions(options) - - # Load the job store + jobStore = Toil.resumeJobStore(config.jobStore) if options.printJobInfo: - printContentsOfJobStore(jobStorePath=options.jobStore, nameOfJob=options.printJobInfo) + printContentsOfJobStore(jobStorePath=config.jobStore, nameOfJob=options.printJobInfo) # TODO: Option to print list of successor jobs # TODO: Option to run job within python debugger, allowing step through of arguments # idea would be to have option to import pdb and set breakpoint at the start of the user's code - # Run the job locally jobID = options.jobID[0] - logger.debug("Going to run the following job locally: %s", jobID) + logger.debug(f"Running the following job locally: {jobID}") workerScript(jobStore, config, jobID, jobID, redirectOutputToLogFile=False) - logger.debug("Ran the following job locally: %s", jobID) + logger.debug(f"Finished running: {jobID}") diff --git a/src/toil/utils/toilDestroyCluster.py b/src/toil/utils/toilDestroyCluster.py index f291662099..86d4025ba1 100644 --- a/src/toil/utils/toilDestroyCluster.py +++ b/src/toil/utils/toilDestroyCluster.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,19 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Terminates the specified cluster and associated resources -""" -from toil.lib.bioio import getBasicOptionParser, parseBasicOptions -from toil.provisioners import clusterFactory -from toil.utils import addBasicProvisionerOptions +"""Terminates the specified cluster and associated resources.""" +from toil.common import parser_with_common_options +from toil.provisioners import cluster_factory +from toil.statsAndLogging import set_logging_from_options def main(): - parser = getBasicOptionParser() - parser = addBasicProvisionerOptions(parser) - config = parseBasicOptions(parser) - cluster = clusterFactory(provisioner=config.provisioner, - clusterName=config.clusterName, - zone=config.zone) + parser = parser_with_common_options(provisioner_options=True, jobstore_option=False) + options = parser.parse_args() + set_logging_from_options(options) + cluster = cluster_factory(provisioner=options.provisioner, + clusterName=options.clusterName, + zone=options.zone) cluster.destroyCluster() diff --git a/src/toil/utils/toilKill.py b/src/toil/utils/toilKill.py index 7509d993b6..ebaf236d16 100644 --- a/src/toil/utils/toilKill.py +++ b/src/toil/utils/toilKill.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,26 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Kills rogue toil processes.""" import logging import os import signal -from toil.common import Config, Toil, jobStoreLocatorHelp -from toil.lib.bioio import getBasicOptionParser, parseBasicOptions -from toil.version import version +from toil.common import Config, Toil, parser_with_common_options +from toil.statsAndLogging import set_logging_from_options -logger = logging.getLogger( __name__ ) +logger = logging.getLogger(__name__) -def main(): - parser = getBasicOptionParser() - parser.add_argument("jobStore", type=str, - help="The location of the job store used by the workflow whose jobs should " - "be killed." + jobStoreLocatorHelp) - parser.add_argument("--version", action='version', version=version) - options = parseBasicOptions(parser) +def main(): + parser = parser_with_common_options() + options = parser.parse_args() + set_logging_from_options(options) config = Config() config.setOptions(options) config.jobStore = config.jobStore[5:] if config.jobStore.startswith('file:') else config.jobStore @@ -39,7 +34,7 @@ def main(): if ':' in config.jobStore: jobStore = Toil.resumeJobStore(config.jobStore) logger.info("Starting routine to kill running jobs in the toil workflow: %s", config.jobStore) - # TODO: This behaviour is now broken src: https://github.com/DataBiosphere/toil/commit/a3d65fc8925712221e4cda116d1825d4a1e963a1 + # TODO: This behaviour is now broken: https://github.com/DataBiosphere/toil/commit/a3d65fc8925712221e4cda116d1825d4a1e963a1 batchSystem = Toil.createBatchSystem(jobStore.config) # Should automatically kill existing jobs, so we're good. for jobID in batchSystem.getIssuedBatchJobIDs(): # Just in case we do it again. batchSystem.killBatchJobs(jobID) diff --git a/src/toil/utils/toilLaunchCluster.py b/src/toil/utils/toilLaunchCluster.py index d64ea9fb87..a9a9a8f060 100644 --- a/src/toil/utils/toilLaunchCluster.py +++ b/src/toil/utils/toilLaunchCluster.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2020 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,31 +11,28 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Launches a toil leader instance with the specified provisioner. -""" +"""Launches a toil leader instance with the specified provisioner.""" import logging +import os from toil import applianceSelf -from toil.lib.bioio import getBasicOptionParser, parseBasicOptions -from toil.provisioners import clusterFactory -from toil.provisioners.aws import checkValidNodeTypes -from toil.utils import addBasicProvisionerOptions, getZoneFromEnv +from toil.common import parser_with_common_options +from toil.provisioners import check_valid_node_types, cluster_factory +from toil.statsAndLogging import set_logging_from_options logger = logging.getLogger(__name__) -def createTagsDict(tagList): - tagsDict = dict() - for tag in tagList: +def create_tags_dict(tags: list) -> dict: + tags_dict = dict() + for tag in tags: key, value = tag.split('=') - tagsDict[key] = value - return tagsDict + tags_dict[key] = value + return tags_dict def main(): - parser = getBasicOptionParser() - parser = addBasicProvisionerOptions(parser) + parser = parser_with_common_options(provisioner_options=True, jobstore_option=False) parser.add_argument("--leaderNodeType", dest="leaderNodeType", required=True, help="Non-preemptable node type to use for the cluster leader.") parser.add_argument("--keyPairName", dest='keyPairName', @@ -93,65 +90,55 @@ def main(): help="Any additional security groups to attach to EC2 instances. Note that a security group " "with its name equal to the cluster name will always be created, thus ensure that " "the extra security groups do not have the same name as the cluster name.") - config = parseBasicOptions(parser) - tags = createTagsDict(config.tags) if config.tags else dict() - checkValidNodeTypes(config.provisioner, config.nodeTypes) - checkValidNodeTypes(config.provisioner, config.leaderNodeType) + options = parser.parse_args() + set_logging_from_options(options) + tags = create_tags_dict(options.tags) if options.tags else dict() + worker_node_types = options.nodeTypes.split(',') if options.nodeTypes else [] + worker_quantities = options.workers.split(',') if options.workers else [] + check_valid_node_types(options.provisioner, worker_node_types + [options.leaderNodeType]) # checks the validity of TOIL_APPLIANCE_SELF before proceeding - applianceSelf(forceDockerAppliance=config.forceDockerAppliance) - - spotBids = [] - nodeTypes = [] - preemptableNodeTypes = [] - numNodes = [] - numPreemptableNodes = [] - if (config.nodeTypes or config.workers) and not (config.nodeTypes and config.workers): - raise RuntimeError("The --nodeTypes and --workers options must be specified together,") - if config.nodeTypes: - nodeTypesList = config.nodeTypes.split(",") - numWorkersList = config.workers.split(",") - if not len(nodeTypesList) == len(numWorkersList): - raise RuntimeError("List of node types must be the same length as the list of workers.") - for nodeTypeStr, num in zip(nodeTypesList, numWorkersList): - parsedBid = nodeTypeStr.split(':', 1) - if len(nodeTypeStr) != len(parsedBid[0]): - #Is a preemptable node - preemptableNodeTypes.append(parsedBid[0]) - spotBids.append(float(parsedBid[1])) - numPreemptableNodes.append(int(num)) - else: - nodeTypes.append(nodeTypeStr) - numNodes.append(int(num)) - - owner = config.owner or config.keyPairName or 'toil' + applianceSelf(forceDockerAppliance=options.forceDockerAppliance) + + owner = options.owner or options.keyPairName or 'toil' # Check to see if the user specified a zone. If not, see if one is stored in an environment variable. - config.zone = config.zone or getZoneFromEnv(config.provisioner) + options.zone = options.zone or os.environ.get(f'TOIL_{options.provisioner.upper()}_ZONE') - if not config.zone: - raise RuntimeError('Please provide a value for --zone or set a default in the TOIL_' + - config.provisioner.upper() + '_ZONE environment variable.') + if not options.zone: + raise RuntimeError(f'Please provide a value for --zone or set a default in the ' + f'TOIL_{options.provisioner.upper()}_ZONE environment variable.') - cluster = clusterFactory(provisioner=config.provisioner, - clusterName=config.clusterName, - zone=config.zone, - nodeStorage=config.nodeStorage) + if (options.nodeTypes or options.workers) and not (options.nodeTypes and options.workers): + raise RuntimeError("The --nodeTypes and --workers options must be specified together.") - cluster.launchCluster(leaderNodeType=config.leaderNodeType, - leaderStorage=config.leaderStorage, - owner=owner, - keyName=config.keyPairName, - botoPath=config.botoPath, - userTags=tags, - vpcSubnet=config.vpcSubnet, - awsEc2ProfileArn=config.awsEc2ProfileArn, - awsEc2ExtraSecurityGroupIds=config.awsEc2ExtraSecurityGroupIds) + if not len(worker_node_types) == len(worker_quantities): + raise RuntimeError("List of node types must be the same length as the list of workers.") - for nodeType, workers in zip(nodeTypes, numNodes): - cluster.addNodes(nodeType=nodeType, numNodes=workers, preemptable=False) - for nodeType, workers, spotBid in zip(preemptableNodeTypes, numPreemptableNodes, spotBids): - cluster.addNodes(nodeType=nodeType, numNodes=workers, preemptable=True, - spotBid=spotBid) + cluster = cluster_factory(provisioner=options.provisioner, + clusterName=options.clusterName, + zone=options.zone, + nodeStorage=options.nodeStorage) + cluster.launchCluster(leaderNodeType=options.leaderNodeType, + leaderStorage=options.leaderStorage, + owner=owner, + keyName=options.keyPairName, + botoPath=options.botoPath, + userTags=tags, + vpcSubnet=options.vpcSubnet, + awsEc2ProfileArn=options.awsEc2ProfileArn, + awsEc2ExtraSecurityGroupIds=options.awsEc2ExtraSecurityGroupIds) + + for worker_node_type, num_workers in zip(worker_node_types, worker_quantities): + if ':' in worker_node_type: + worker_node_type, bid = worker_node_type.split(':', 1) + cluster.addNodes(nodeType=worker_node_type, + numNodes=int(num_workers), + preemptable=True, + spotBid=float(bid)) + else: + cluster.addNodes(nodeType=worker_node_type, + numNodes=int(num_workers), + preemptable=False) diff --git a/src/toil/utils/toilMain.py b/src/toil/utils/toilMain.py index 7a1c9320e8..f0af13a914 100755 --- a/src/toil/utils/toilMain.py +++ b/src/toil/utils/toilMain.py @@ -31,9 +31,15 @@ def main(): def loadModules(): # noinspection PyUnresolvedReferences - from toil.utils import (toilClean, toilDebugFile, toilDebugJob, # noqa - toilDestroyCluster, toilKill, toilLaunchCluster, - toilRsyncCluster, toilSshCluster, toilStats, + from toil.utils import (toilClean, + toilDebugFile, + toilDebugJob, + toilDestroyCluster, + toilKill, + toilLaunchCluster, + toilRsyncCluster, + toilSshCluster, + toilStats, toilStatus) return {"-".join([i.lower() for i in re.findall('[A-Z][^A-Z]*', name)]): module for name, module in locals().items()} diff --git a/src/toil/utils/toilRsyncCluster.py b/src/toil/utils/toilRsyncCluster.py index af557e29d4..ebe0dc2845 100644 --- a/src/toil/utils/toilRsyncCluster.py +++ b/src/toil/utils/toilRsyncCluster.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,22 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Rsyncs into the toil appliance container running on the leader of the cluster -""" +"""Rsyncs into the toil appliance container running on the leader of the cluster.""" import argparse import logging -from toil.lib.bioio import getBasicOptionParser, parseBasicOptions -from toil.provisioners import clusterFactory -from toil.utils import addBasicProvisionerOptions +from toil.common import parser_with_common_options +from toil.provisioners import cluster_factory +from toil.statsAndLogging import set_logging_from_options logger = logging.getLogger(__name__) def main(): - parser = getBasicOptionParser() - parser = addBasicProvisionerOptions(parser) + parser = parser_with_common_options(provisioner_options=True, jobstore_option=False) parser.add_argument("--insecure", dest='insecure', action='store_true', required=False, help="Temporarily disable strict host key checking.") parser.add_argument("args", nargs=argparse.REMAINDER, help="Arguments to pass to" @@ -35,8 +32,9 @@ def main(): " specify `toil rsync-cluster -p aws test-cluster example.py :`." "\nOr, to download a file from the remote:, `toil rsync-cluster" " -p aws test-cluster :example.py .`") - config = parseBasicOptions(parser) - cluster = clusterFactory(provisioner=config.provisioner, - clusterName=config.clusterName, - zone=config.zone) - cluster.getLeader().coreRsync(args=config.args, strict=not config.insecure) + options = parser.parse_args() + set_logging_from_options(options) + cluster = cluster_factory(provisioner=options.provisioner, + clusterName=options.clusterName, + zone=options.zone) + cluster.getLeader().coreRsync(args=options.args, strict=not options.insecure) diff --git a/src/toil/utils/toilSshCluster.py b/src/toil/utils/toilSshCluster.py index 50c84b5994..c20d46e7c4 100644 --- a/src/toil/utils/toilSshCluster.py +++ b/src/toil/utils/toilSshCluster.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,32 +11,30 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -SSHs into the toil appliance container running on the leader of the cluster -""" +"""SSH into the toil appliance container running on the leader of the cluster.""" import argparse import logging import sys -from toil.lib.bioio import getBasicOptionParser, parseBasicOptions -from toil.provisioners import clusterFactory -from toil.utils import addBasicProvisionerOptions +from toil.common import parser_with_common_options +from toil.provisioners import cluster_factory +from toil.statsAndLogging import set_logging_from_options logger = logging.getLogger(__name__) def main(): - parser = getBasicOptionParser() - parser = addBasicProvisionerOptions(parser) + parser = parser_with_common_options(provisioner_options=True, jobstore_option=False) parser.add_argument("--insecure", action='store_true', help="Temporarily disable strict host key checking.") parser.add_argument("--sshOption", dest='sshOptions', default=[], action='append', help="Pass an additional option to the SSH command.") parser.add_argument('args', nargs=argparse.REMAINDER) - config = parseBasicOptions(parser) - cluster = clusterFactory(provisioner=config.provisioner, - clusterName=config.clusterName, - zone=config.zone) - command = config.args if config.args else ['bash'] - cluster.getLeader().sshAppliance(*command, strict=not config.insecure, tty=sys.stdin.isatty(), - sshOptions=config.sshOptions) + options = parser.parse_args() + set_logging_from_options(options) + cluster = cluster_factory(provisioner=options.provisioner, + clusterName=options.clusterName, + zone=options.zone) + command = options.args if options.args else ['bash'] + cluster.getLeader().sshAppliance(*command, strict=not options.insecure, tty=sys.stdin.isatty(), + sshOptions=options.sshOptions) diff --git a/src/toil/utils/toilStats.py b/src/toil/utils/toilStats.py index 23b13857d6..21cbcdf286 100644 --- a/src/toil/utils/toilStats.py +++ b/src/toil/utils/toilStats.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,24 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -""" -Reports statistical data about a given Toil workflow. -""" - +"""Reports statistical data about a given Toil workflow.""" import json import logging from functools import partial -from toil.common import Config, Toil, jobStoreLocatorHelp -from toil.lib.bioio import getBasicOptionParser, parseBasicOptions +from toil.common import Config, Toil, parser_with_common_options from toil.lib.expando import Expando -from toil.version import version +from toil.statsAndLogging import set_logging_from_options -logger = logging.getLogger( __name__ ) +logger = logging.getLogger(__name__) -class ColumnWidths(object): +class ColumnWidths: """ Convenience object that stores the width of columns for printing. Helps make things pretty. """ @@ -56,81 +51,17 @@ def report(self): for f in self.fields: print('%s %s %d' % (c, f, self.getWidth(c, f))) -def initializeOptions(parser): - parser.add_argument("jobStore", type=str, - help="The location of the job store used by the workflow for which " - "statistics should be reported. " + jobStoreLocatorHelp) - parser.add_argument("--outputFile", dest="outputFile", default=None, - help="File in which to write results") - parser.add_argument("--raw", action="store_true", default=False, - help="output the raw json data.") - parser.add_argument("--pretty", "--human", action="store_true", default=False, - help=("if not raw, prettify the numbers to be " - "human readable.")) - parser.add_argument("--categories", - help=("comma separated list from [time, clock, wait, " - "memory]")) - parser.add_argument("--sortCategory", default="time", - help=("how to sort Job list. may be from [alpha, " - "time, clock, wait, memory, count]. " - "default=%(default)s")) - parser.add_argument("--sortField", default="med", - help=("how to sort Job list. may be from [min, " - "med, ave, max, total]. " - "default=%(default)s")) - parser.add_argument("--sortReverse", "--reverseSort", default=False, - action="store_true", - help="reverse sort order.") - parser.add_argument("--version", action='version', version=version) - -def checkOptions(options, parser): - """ Check options, throw parser.error() if something goes wrong - """ - - if options.jobStore == None: - parser.error("Specify --jobStore") - defaultCategories = ["time", "clock", "wait", "memory"] - if options.categories is None: - options.categories = defaultCategories - else: - options.categories = [x.lower() for x in options.categories.split(",")] - for c in options.categories: - if c not in defaultCategories: - parser.error("Unknown category %s. Must be from %s" - % (c, str(defaultCategories))) - extraSort = ["count", "alpha"] - if options.sortCategory is not None: - if (options.sortCategory not in defaultCategories and - options.sortCategory not in extraSort): - parser.error("Unknown --sortCategory %s. Must be from %s" - % (options.sortCategory, - str(defaultCategories + extraSort))) - sortFields = ["min", "med", "ave", "max", "total"] - if options.sortField is not None: - if (options.sortField not in sortFields): - parser.error("Unknown --sortField %s. Must be from %s" - % (options.sortField, str(sortFields))) - -def printJson(elem): - """ Return a JSON formatted string - """ - prettyString = json.dumps(elem, indent=4, separators=(',',': ')) - return prettyString def padStr(s, field=None): - """ Pad the begining of a string with spaces, if necessary. - """ - if field is None: + """Pad the beginning of a string with spaces, if necessary.""" + if field is None or len(s) >= field: return s else: - if len(s) >= field: - return s - else: - return " " * (field - len(s)) + s + return " " * (field - len(s)) + s + def prettyMemory(k, field=None, isBytes=False): - """ Given input k as kilobytes, return a nicely formatted string. - """ + """Given input k as kilobytes, return a nicely formatted string.""" if isBytes: k /= 1024 if k < 1024: @@ -144,6 +75,7 @@ def prettyMemory(k, field=None, isBytes=False): if k < (1024 * 1024 * 1024 * 1024 * 1024): return padStr("%.1fP" % (k / 1024.0 / 1024.0 / 1024.0 / 1024.0), field) + def prettyTime(t, field=None): """ Given input t as seconds, return a nicely formatted string. """ @@ -171,12 +103,10 @@ def prettyTime(t, field=None): return padStr("%dday%s%dh%dm%ds" % (d, dPlural, h, m, s), field) w = floor(t / 7. / 24. / 60. / 60.) d = floor((t - (w * 7 * 24 * 60 * 60)) / 24. / 60. / 60.) - h = floor((t - - (w * 7. * 24. * 60. * 60.) + h = floor((t - (w * 7. * 24. * 60. * 60.) - (d * 24. * 60. * 60.)) - / 60. / 60.) - m = floor((t - - (w * 7. * 24. * 60. * 60.) + / 60. / 60.) + m = floor((t - (w * 7. * 24. * 60. * 60.) - (d * 24. * 60. * 60.) - (h * 60. * 60.)) / 60.) s = t % 60 @@ -185,20 +115,18 @@ def prettyTime(t, field=None): return padStr("%dweek%s%dday%s%dh%dm%ds" % (w, wPlural, d, dPlural, h, m, s), field) + def reportTime(t, options, field=None): - """ Given t seconds, report back the correct format as string. - """ + """Given t seconds, report back the correct format as string.""" if options.pretty: return prettyTime(t, field=field) - else: - if field is not None: - return "%*.2f" % (field, t) - else: - return "%.2f" % t + elif field is not None: + return "%*.2f" % (field, t) + return "%.2f" % t + def reportMemory(k, options, field=None, isBytes=False): - """ Given k kilobytes, report back the correct format as string. - """ + """Given k kilobytes, report back the correct format as string.""" if options.pretty: return prettyMemory(int(k), field=field, isBytes=isBytes) else: @@ -209,24 +137,11 @@ def reportMemory(k, options, field=None, isBytes=False): else: return "%dK" % int(k) -def reportNumber(n, options, field=None): - """ Given n an integer, report back the correct format as string. - """ - if field is not None: - return "%*g" % (field, n) - else: - return "%g" % n -def refineData(root, options): - """ walk down from the root and gather up the important bits. - """ - worker = root.worker - job = root.jobs - jobTypesTree = root.job_types - jobTypes = [] - for childName in jobTypesTree: - jobTypes.append(jobTypesTree[childName]) - return root, worker, job, jobTypes +def reportNumber(n, field=None): + """Given n an integer, report back the correct format as string.""" + return "%*g" % (field, n) if field else "%g" % n + def sprintTag(key, tag, options, columnWidths=None): """ Generate a pretty-print ready string from a JTTag(). @@ -235,7 +150,7 @@ def sprintTag(key, tag, options, columnWidths=None): columnWidths = ColumnWidths() header = " %7s " % decorateTitle("Count", options) sub_header = " %7s " % "n" - tag_str = " %s" % reportNumber(tag.total_number, options, field=7) + tag_str = f" {reportNumber(n=tag.total_number, field=7)}" out_str = "" if key == "job": out_str += " %-12s | %7s%7s%7s%7s\n" % ("Worker Jobs", "min", @@ -243,7 +158,7 @@ def sprintTag(key, tag, options, columnWidths=None): worker_str = "%s| " % (" " * 14) for t in [tag.min_number_per_worker, tag.median_number_per_worker, tag.average_number_per_worker, tag.max_number_per_worker]: - worker_str += reportNumber(t, options, field=7) + worker_str += reportNumber(n=t, field=7) out_str += worker_str + "\n" if "time" in options.categories: header += "| %*s " % (columnWidths.title("time"), @@ -336,22 +251,17 @@ def decorateSubHeader(title, columnWidths, options): s += " " return s + def get(tree, name): - """ Return a float value attribute NAME from TREE. - """ - if name in tree: - value = tree[name] - else: - return float("nan") + """Return a float value attribute NAME from TREE.""" try: - a = float(value) + return float(tree.get(name, "nan")) except ValueError: - a = float("nan") - return a + return float("nan") + def sortJobs(jobTypes, options): - """ Return a jobTypes all sorted. - """ + """Return a jobTypes all sorted.""" longforms = {"med": "median", "ave": "average", "min": "min", @@ -376,15 +286,15 @@ def sortJobs(jobTypes, options): return sorted(jobTypes, key=lambda tag: tag.total_number, reverse=options.sortReverse) + def reportPrettyData(root, worker, job, job_types, options): - """ print the important bits out. - """ + """Print the important bits out.""" out_str = "Batch System: %s\n" % root.batch_system out_str += ("Default Cores: %s Default Memory: %s\n" "Max Cores: %s\n" % ( - reportNumber(get(root, "default_cores"), options), + reportNumber(n=get(root, "default_cores")), reportMemory(get(root, "default_memory"), options, isBytes=True), - reportNumber(get(root, "max_cores"), options), + reportNumber(n=get(root, "max_cores")), )) out_str += ("Total Clock: %s Total Runtime: %s\n" % ( reportTime(get(root, "total_clock"), options), @@ -401,6 +311,7 @@ def reportPrettyData(root, worker, job, job_types, options): out_str += sprintTag(t.name, t, options, columnWidths=columnWidths) return out_str + def computeColumnWidths(job_types, worker, job, options): """ Return a ColumnWidths() object with the correct max widths. """ @@ -411,6 +322,7 @@ def computeColumnWidths(job_types, worker, job, options): updateColumnWidths(job, cw, options) return cw + def updateColumnWidths(tag, cw, options): """ Update the column width attributes for this tag's fields. """ @@ -433,6 +345,7 @@ def updateColumnWidths(tag, cw, options): # this string is larger than max, width must be increased cw.setWidth(category, field, len(s) + 1) + def buildElement(element, items, itemName): """ Create an element for output. """ @@ -494,6 +407,7 @@ def assertNonnegative(i,name): ) return element[itemName] + def createSummary(element, containingItems, containingItemName, getFn): itemCounts = [len(getFn(containingItem)) for containingItem in containingItems] @@ -573,13 +487,13 @@ def fn4(job): collatedStatsTag.name = "collatedStatsTag" return collatedStatsTag + def reportData(tree, options): # Now dump it all out to file if options.raw: - out_str = printJson(tree) + out_str = json.dumps(tree, indent=4, separators=(',', ': ')) else: - root, worker, job, job_types = refineData(tree, options) - out_str = reportPrettyData(root, worker, job, job_types, options) + out_str = reportPrettyData(tree, tree.worker, tree.jobs, tree.job_types.values(), options) if options.outputFile is not None: fileHandle = open(options.outputFile, "w") fileHandle.write(out_str) @@ -587,13 +501,38 @@ def reportData(tree, options): # Now dump onto the screen print(out_str) + +category_choices = ["time", "clock", "wait", "memory"] +sort_category_choices = ["time", "clock", "wait", "memory", "alpha", "count"] +sort_field_choices = ['min', 'med', 'ave', 'max', 'total'] + + +def add_stats_options(parser): + parser.add_argument("--outputFile", dest="outputFile", default=None, help="File in which to write results.") + parser.add_argument("--raw", action="store_true", default=False, help="Return raw json data.") + parser.add_argument("--pretty", "--human", action="store_true", default=False, + help="if not raw, prettify the numbers to be human readable.") + parser.add_argument("--sortReverse", "--reverseSort", default=False, action="store_true", help="Reverse sort.") + parser.add_argument("--categories", default=','.join(category_choices), type=str, + help=f"Comma separated list of any of the following: {category_choices}.") + parser.add_argument("--sortCategory", default="time", choices=sort_category_choices, + help=f"How to sort job categories. Choices: {sort_category_choices}. Default: time.") + parser.add_argument("--sortField", default="med", choices=sort_field_choices, + help=f"How to sort job fields. Choices: {sort_field_choices}. Default: med.") + + def main(): - """ Reports stats on the workflow, use with --stats option to toil. - """ - parser = getBasicOptionParser() - initializeOptions(parser) - options = parseBasicOptions(parser) - checkOptions(options, parser) + """Reports stats on the workflow, use with --stats option to toil.""" + parser = parser_with_common_options() + add_stats_options(parser) + options = parser.parse_args() + + for c in options.categories.split(","): + if c.strip() not in category_choices: + raise ValueError(f'{c} not in {category_choices}!') + options.categories = [x.strip().lower() for x in options.categories.split(",")] + + set_logging_from_options(options) config = Config() config.setOptions(options) jobStore = Toil.resumeJobStore(config.jobStore) diff --git a/src/toil/utils/toilStatus.py b/src/toil/utils/toilStatus.py index 36b41b4b56..509967e412 100644 --- a/src/toil/utils/toilStatus.py +++ b/src/toil/utils/toilStatus.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,28 +11,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -"""Tool for reporting on job status. -""" - -# standard library +"""Tool for reporting on job status.""" import logging import os import sys from functools import reduce -# toil imports -from toil.common import Config, Toil, jobStoreLocatorHelp +from toil.common import Config, Toil, parser_with_common_options from toil.job import JobException, ServiceJobDescription from toil.jobStores.abstractJobStore import (NoSuchFileException, NoSuchJobStoreException) -from toil.lib.bioio import getBasicOptionParser, parseBasicOptions -from toil.statsAndLogging import StatsAndLogging -from toil.version import version +from toil.statsAndLogging import StatsAndLogging, set_logging_from_options logger = logging.getLogger(__name__) -class ToilStatus(): + +class ToilStatus: """Tool for reporting on job status.""" def __init__(self, jobStoreName, specifiedJobs=None): self.jobStoreName = jobStoreName @@ -60,7 +54,7 @@ def print_dot_chart(self): # Print the edges for job in set(self.jobsToReport): - for level, jobList in enumerate(job.stack): + for level, jobList in enumerate(job.stack): for childJob in jobList: # Check, b/c successor may be finished / not in the set of jobs if childJob.jobStoreID in jobsToNodeNames: @@ -73,14 +67,12 @@ def printJobLog(self): """Takes a list of jobs, finds their log files, and prints them to the terminal.""" for job in self.jobsToReport: if job.logJobStoreFileID is not None: - # TODO: This looks intended to be machine-readable, but the format is - # unspecified and no escaping is done. But keep these tags around. - msg = "LOG_FILE_OF_JOB:%s LOG:" % job with job.getLogFileHandle(self.jobStore) as fH: - msg += StatsAndLogging.formatLogStream(fH) - print(msg) + # TODO: This looks intended to be machine-readable, but the format is + # unspecified and no escaping is done. But keep these tags around. + print(StatsAndLogging.formatLogStream(fH, job_name=f"LOG_FILE_OF_JOB:{job} LOG:")) else: - print("LOG_FILE_OF_JOB:%s LOG: Job has no log file" % job) + print(f"LOG_FILE_OF_JOB: {job} LOG: Job has no log file") def printJobChildren(self): """Takes a list of jobs, and prints their successors.""" @@ -179,7 +171,7 @@ def getPIDStatus(jobStoreName): except NoSuchFileException: pass return 'QUEUED' - + @staticmethod def getStatus(jobStoreName): """ @@ -285,14 +277,10 @@ def traverseJobGraph(self, rootJob, jobsToReport=None, foundJobStoreIDs=None): return jobsToReport + def main(): """Reports the state of a Toil workflow.""" - parser = getBasicOptionParser() - - parser.add_argument("jobStore", type=str, - help="The location of a job store that holds the information about the " - "workflow whose status is to be reported on." + jobStoreLocatorHelp) - + parser = parser_with_common_options() parser.add_argument("--failIfNotComplete", action="store_true", help="Return exit value of 1 if toil jobs not all completed. default=%(default)s", default=False) @@ -322,9 +310,8 @@ def main(): help="Print children of each job. default=%(default)s", default=False) - parser.add_argument("--version", action='version', version=version) - - options = parseBasicOptions(parser) + options = parser.parse_args() + set_logging_from_options(options) if len(sys.argv) == 1: parser.print_help() @@ -363,14 +350,14 @@ def main(): status.print_dot_chart() if options.stats: print('Of the %i jobs considered, ' - 'there are %i jobs with children, ' - '%i jobs ready to run, ' - '%i zombie jobs, ' - '%i jobs with services, ' - '%i services, ' - 'and %i jobs with log files currently in %s.' % - (len(status.jobsToReport), len(hasChildren), len(readyToRun), len(zombies), - len(hasServices), len(services), len(hasLogFile), status.jobStore)) + 'there are %i jobs with children, ' + '%i jobs ready to run, ' + '%i zombie jobs, ' + '%i jobs with services, ' + '%i services, ' + 'and %i jobs with log files currently in %s.' % + (len(status.jobsToReport), len(hasChildren), len(readyToRun), len(zombies), + len(hasServices), len(services), len(hasLogFile), status.jobStore)) if len(status.jobsToReport) > 0 and options.failIfNotComplete: # Upon workflow completion, all jobs will have been removed from job store diff --git a/src/toil/utils/toilUpdateEC2Instances.py b/src/toil/utils/toilUpdateEC2Instances.py index 011cf21993..68f284dca2 100644 --- a/src/toil/utils/toilUpdateEC2Instances.py +++ b/src/toil/utils/toilUpdateEC2Instances.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,35 +11,29 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Updates Toil's internal list of EC2 instance types. -""" +"""Updates Toil's internal list of EC2 instance types.""" import logging import socket from toil.lib.ec2nodes import updateStaticEC2Instances -logger = logging.getLogger( __name__ ) +logger = logging.getLogger(__name__) -def internetConnection(): - """ - Returns True if there is an internet connection present, and False otherwise. - - :return: - """ +def internet_connection() -> bool: + """Returns True if there is an internet connection present, and False otherwise.""" try: socket.create_connection(("www.stackoverflow.com", 80)) return True except OSError: - pass - return False + return False def main(): - if not internetConnection(): + if not internet_connection(): raise RuntimeError('No internet. Updating the EC2 Instance list requires internet.') updateStaticEC2Instances() -if __name__=="__main__": + +if __name__ == "__main__": main() diff --git a/src/toil/wdl/wdl_analysis.py b/src/toil/wdl/wdl_analysis.py index 3adf2daf69..5647f4236d 100644 --- a/src/toil/wdl/wdl_analysis.py +++ b/src/toil/wdl/wdl_analysis.py @@ -17,17 +17,14 @@ from collections import OrderedDict import toil.wdl.wdl_parser as wdl_parser - -from toil.wdl.wdl_types import ( - WDLStringType, - WDLIntType, - WDLFloatType, - WDLBooleanType, - WDLFileType, - WDLArrayType, - WDLPairType, - WDLMapType -) +from toil.wdl.wdl_types import (WDLArrayType, + WDLBooleanType, + WDLFileType, + WDLFloatType, + WDLIntType, + WDLMapType, + WDLPairType, + WDLStringType) wdllogger = logging.getLogger(__name__) diff --git a/src/toil/wdl/wdl_functions.py b/src/toil/wdl/wdl_functions.py index 2879191bb0..181e40d8fa 100644 --- a/src/toil/wdl/wdl_functions.py +++ b/src/toil/wdl/wdl_functions.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018-2020 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,10 +24,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union from toil.fileStores.abstractFileStore import AbstractFileStore -from toil.wdl.wdl_types import ( - WDLFile, - WDLPair -) +from toil.wdl.wdl_types import WDLFile, WDLPair wdllogger = logging.getLogger(__name__) diff --git a/src/toil/wdl/wdl_synthesis.py b/src/toil/wdl/wdl_synthesis.py index eed0a991d7..9761fb57a7 100644 --- a/src/toil/wdl/wdl_synthesis.py +++ b/src/toil/wdl/wdl_synthesis.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018-2020 UCSC Computational Genomics Lab +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,14 +16,12 @@ import toil.wdl.wdl_parser as wdl_parser from toil.wdl.wdl_functions import heredoc_wdl -from toil.wdl.wdl_types import ( - WDLType, - WDLCompoundType, - WDLFileType, - WDLArrayType, - WDLPairType, - WDLMapType -) +from toil.wdl.wdl_types import (WDLArrayType, + WDLCompoundType, + WDLFileType, + WDLMapType, + WDLPairType, + WDLType) wdllogger = logging.getLogger(__name__) diff --git a/src/toil/worker.py b/src/toil/worker.py index 5cc6bc6744..84aea54df8 100644 --- a/src/toil/worker.py +++ b/src/toil/worker.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -33,9 +33,10 @@ from toil.deferred import DeferredFunctionManager from toil.fileStores.abstractFileStore import AbstractFileStore from toil.job import CheckpointJobDescription, Job -from toil.lib.bioio import (configureRootLogger, getTotalCpuTime, - getTotalCpuTimeAndMemoryUsage, setLogLevel) from toil.lib.expando import MagicExpando +from toil.lib.resources import (get_total_cpu_time, + get_total_cpu_time_and_memory_usage) +from toil.statsAndLogging import configure_root_logger, set_log_level try: from toil.cwl.cwltoil import CWL_INTERNAL_JOBS @@ -130,8 +131,8 @@ def workerScript(jobStore, config, jobName, jobStoreID, redirectOutputToLogFile= :return int: 1 if a job failed, or 0 if all jobs succeeded """ - configureRootLogger() - setLogLevel(config.logLevel) + configure_root_logger() + set_log_level(config.logLevel) ########################################## #Create the worker killer, if requested @@ -350,7 +351,7 @@ def workerScript(jobStore, config, jobName, jobStoreID, redirectOutputToLogFile= ########################################## if config.stats: - startClock = getTotalCpuTime() + startClock = get_total_cpu_time() startTime = time.time() while True: @@ -478,7 +479,7 @@ def workerScript(jobStore, config, jobName, jobStoreID, redirectOutputToLogFile= #Finish up the stats ########################################## if config.stats: - totalCPUTime, totalMemoryUsage = getTotalCpuTimeAndMemoryUsage() + totalCPUTime, totalMemoryUsage = get_total_cpu_time_and_memory_usage() statsDict.workers.time = str(time.time() - startTime) statsDict.workers.clock = str(totalCPUTime - startClock) statsDict.workers.memory = str(totalMemoryUsage) diff --git a/version_template.py b/version_template.py index d7cbd59aa7..faacc9d061 100644 --- a/version_template.py +++ b/version_template.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 Regents of the University of California +# Copyright (C) 2015-2021 Regents of the University of California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """This script is a template for src/toil/version.py. Running it without arguments echoes all globals, i.e. module attributes. Constant assignments will be echoed verbatim while callables will be invoked and their result echoed as an assignment using the function name as the left-hand