From d5786fc68c1f8a4faccb038fa7dd6a5d3869f63d Mon Sep 17 00:00:00 2001 From: bio-boris Date: Wed, 7 Aug 2024 10:13:44 -0500 Subject: [PATCH] D->M 0.0.14 (#482) * Update RELEASE_NOTES.md * add dependabot.yml && fix .pre-commit-config.yaml file (#478) * add dependabot * fix pre-comit config * black format fix * flake8 fix * SECURITY-36 3.10.10 (#477) * Testing upgrades * Update mongo version * Update python to 3.10.10 * Update ee2-tests.yml * SECURITY-36 Mongo3.6/Mongo7 Matrix (#484) * Linting * Bump version and release notes --------- Co-authored-by: Boris * Bump actions/setup-python from 4 to 5 in /.github/workflows (#481) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: Sijie Xiang Co-authored-by: Boris Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/dependabot.yml | 23 ++ .github/workflows/check_build.yml | 21 ++ .github/workflows/ee2-tests.yml | 42 ++-- .pre-commit-config.yaml | 4 +- Dockerfile | 17 +- Makefile | 2 +- RELEASE_NOTES.md | 10 +- docker-compose.yml | 109 +++++---- kbase.yml | 2 +- lib/biokbase/log.py | 229 +++++++++++------- lib/execution_engine2/authclient.py | 41 ++-- lib/execution_engine2/sdk/EE2Runjob.py | 12 +- .../sdk/job_submission_parameters.py | 14 +- .../utils/application_info.py | 2 +- .../utils/job_requirements_resolver.py | 12 +- lib/execution_engine2/utils/user_info.py | 2 +- lib/installed_clients/authclient.py | 2 +- requirements.txt | 9 +- .../mongo-init.js | 3 +- .../mongo-init.js | 14 ++ test/tests_for_auth/ee2_admin_mode_test.py | 10 +- 21 files changed, 352 insertions(+), 228 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/check_build.yml rename test/dockerfiles/mongo/{docker-entrypoint-initdb.d => docker-entrypoint-initdb.d-3.6}/mongo-init.js (82%) create mode 100755 test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/mongo-init.js diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..65216f7b5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,23 @@ +version: 2 +updates: + + # Docker + - package-ecosystem: docker + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 25 + + # Python + - package-ecosystem: "pip" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "monthly" + open-pull-requests-limit: 25 + + # GitHub Actions + - package-ecosystem: "github-actions" + directory: ".github/workflows" + schedule: + interval: "monthly" + open-pull-requests-limit: 25 \ No newline at end of file diff --git a/.github/workflows/check_build.yml b/.github/workflows/check_build.yml new file mode 100644 index 000000000..5f13de013 --- /dev/null +++ b/.github/workflows/check_build.yml @@ -0,0 +1,21 @@ +name: Build EE2 Docker Image + +on: + pull_request + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Build and push + uses: docker/build-push-action@v6 + with: + push: false + tags: ee2/test:test \ No newline at end of file diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index 671644388..41007ff9a 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -1,12 +1,6 @@ # This workflow will install Python dependencies, run tests and lint # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -# To ssh into this build add the following: -#- name: Start SSH session -# uses: luchihoratiu/debug-via-ssh@main -# with: -# NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} -# SSH_PASS: ${{ secrets.SSH_PASS }} name: Execution Engine 2 Test Suite @@ -18,7 +12,7 @@ jobs: runs-on: ubuntu-latest name: Lint With Black steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable with: options: "--check --verbose" @@ -35,11 +29,11 @@ jobs: name: Lint With Flake8 steps: - name: Check out source repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python environment - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.10.14" - name: flake8 Lint Lib uses: py-actions/flake8@v2 with: @@ -50,22 +44,38 @@ jobs: path: "./test" + Build_and_Run_Tests_and_CodeCov: name: Build and Run Tests and CodeCov runs-on: ubuntu-latest + strategy: + matrix: + mongo-config: + - version: "7.0" + init-path: "./test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/" + - version: "3.6" + init-path: "./test/dockerfiles/mongo/docker-entrypoint-initdb.d-3.6/" steps: - name: Check out source repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 + - name: Set up Python environment + uses: actions/setup-python@v5 + with: + python-version: "3.10.14" - name: Install dependencies run: | pip install -r requirements.txt git clone https://github.com/kbase/jars /opt/jars - - name: Build Docker Image + - name: Set MongoDB Version and Init Path run: | - docker build . -t execution_engine2:test - - name: Run Tests + echo "MONGO_VERSION=${{ matrix.mongo-config.version }}" >> $GITHUB_ENV + echo "INIT_PATH=${{ matrix.mongo-config.init-path }}" >> $GITHUB_ENV + - name: Start Services and Run Tests run: | - docker-compose up -d + docker compose up -d cp test/env/test.travis.env test.env make test-coverage - codecov + codecov --token="${{ secrets.CODECOV_TOKEN }}" + - name: Cleanup + run: | + docker compose down diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8468b8c65..efd895b36 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: black exclude: ".*execution_engine2Impl.py" -- repo: https://gitlab.com/pycqa/flake8 - rev: '3.9.2' +- repo: https://github.com/PyCQA/flake8 + rev: 3.9.2 hooks: - id: flake8 diff --git a/Dockerfile b/Dockerfile index baa46f512..76db5f3a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,23 +26,8 @@ RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize rm /tmp/dockerize.tgz -# install mongodb -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 \ - && echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/3.6 main" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends mongodb-org=3.6.11 mongodb-org-server=3.6.11 mongodb-org-shell=3.6.11 mongodb-org-mongos=3.6.11 mongodb-org-tools=3.6.11 \ - && apt-get install -y --no-install-recommends mongodb \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN echo "mongodb-org hold" | dpkg --set-selections \ - && echo "mongodb-org-server hold" | dpkg --set-selections \ - && echo "mongodb-org-shell hold" | dpkg --set-selections \ - && echo "mongodb-org-mongos hold" | dpkg --set-selections \ - && echo "mongodb-org-tools hold" | dpkg --set-selections - #Install Python3 and Libraries (source /root/miniconda/bin/activate) -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \ +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_24.5.0-0-Linux-x86_64.sh -O ~/miniconda.sh \ && bash ~/miniconda.sh -b -p /miniconda-latest # Setup Cron diff --git a/Makefile b/Makefile index 0e3ad8c5e..3eebfda65 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ setup-database: test-coverage: # Set up travis user in mongo @echo "Run tests for $(TESTS)" - PYTHONPATH=.:lib:test pytest --cov-report=xml --cov lib/execution_engine2/ --verbose $(TESTS) + PYTHONPATH=.:lib:test pytest --cov-report=xml --cov-report=term --cov lib/execution_engine2/ --verbose $(TESTS) build-condor-test-image: cd test/dockerfiles/condor && echo `pwd` && docker build -f Dockerfile . -t $(CONDOR_DOCKER_IMAGE_TAG_NAME) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 48484e28e..1460c0a4b 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,8 +1,16 @@ # execution_engine2 (ee2) release notes ========================================= +## 0.0.14 +- Update clients to work with mongo7 by updating pymongo and mongoengine +- Fix linting issues from flake8 and black +- Update outdated libraries that can no longer be pip installed +- The latest Dockerfile build contains conda which installs python 3.12.4 so these changes will pin python to 3.10 and save us headache in the future +- Speed up tests by moving docker build step out +- Create matrix of tests to run against mongo3.6 and mongo7 + ## 0.0.13 -* Bump version in impl file +* Bump version in impl file and sync branches ## 0.0.12 * Forcing black to 22.1.0 to make sure that GHA doesn't suddenly fail diff --git a/docker-compose.yml b/docker-compose.yml index 46896e28c..996708d33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,57 +2,58 @@ version: "3.1" services: + # For Local Dev # Requires a ` docker build . -t execution_engine2:test ` # Requires a ` make database ` - ee2: - image: execution_engine2:test - command: - - "-poll" - - "-template" - - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config" - - "-template" - - "/kb/module/test/deploy.cfg:/kb/module/deploy.cfg" - - "-timeout" - - "120s" - - "-wait" - - "tcp://mongodb:27017" - - "-stdout" - - "/kb/deployment/jettybase/logs/request.log" - - "./scripts/entrypoint.sh" - entrypoint: [ "dockerize" ] - depends_on: ["mongodb","condor"] - environment: - - POOL_PASSWORD=weakpassword - env_file: test/deploy.cfg - volumes: - - ./:/ee2 - - ee2_with_ssh: - image: execution_engine2:test - command: - - "-poll" - - "-template" - - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config" - - "-template" - - "/kb/module/build/templates/deploy.docker.cfg.templ:/kb/module/deploy.cfg" - - "-timeout" - - "120s" - - "-wait" - - "tcp://mongodb:27017" - - "-stdout" - - "/kb/deployment/jettybase/logs/request.log" - - "./scripts/entrypoint.sh" - entrypoint: [ "dockerize" ] - depends_on: ["mongodb","condor"] - environment: - - POOL_PASSWORD=weakpassword - - LOCAL_DEV=1 - - KB_DEPLOYMENT_CONFIG=/kb/module/deploy.cfg - env_file: test/deploy.cfg - ports: ["23:22","5678:5678"] - volumes: - - ./:/ee2 - - /Users:/Users +# ee2: +# image: execution_engine2:test +# command: +# - "-poll" +# - "-template" +# - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config" +# - "-template" +# - "/kb/module/test/deploy.cfg:/kb/module/deploy.cfg" +# - "-timeout" +# - "120s" +# - "-wait" +# - "tcp://mongodb:27017" +# - "-stdout" +# - "/kb/deployment/jettybase/logs/request.log" +# - "./scripts/entrypoint.sh" +# entrypoint: [ "dockerize" ] +# depends_on: ["mongodb","condor"] +# environment: +# - POOL_PASSWORD=weakpassword +# env_file: test/deploy.cfg +# volumes: +# - ./:/ee2 +# +# ee2_with_ssh: +# image: execution_engine2:test +# command: +# - "-poll" +# - "-template" +# - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config" +# - "-template" +# - "/kb/module/build/templates/deploy.docker.cfg.templ:/kb/module/deploy.cfg" +# - "-timeout" +# - "120s" +# - "-wait" +# - "tcp://mongodb:27017" +# - "-stdout" +# - "/kb/deployment/jettybase/logs/request.log" +# - "./scripts/entrypoint.sh" +# entrypoint: [ "dockerize" ] +# depends_on: ["mongodb","condor"] +# environment: +# - POOL_PASSWORD=weakpassword +# - LOCAL_DEV=1 +# - KB_DEPLOYMENT_CONFIG=/kb/module/deploy.cfg +# env_file: test/deploy.cfg +# ports: ["23:22","5678:5678"] +# volumes: +# - ./:/ee2 +# - /Users:/Users @@ -146,13 +147,15 @@ services: ZOOKEEPER_SYNC_LIMIT: 2docker-compose rm ZOOKEEPER_SERVERS: zookeeper:12888:13888 + mongodb: - image: mongo:3.2 + image: mongo:${MONGO_VERSION:-7.0} # Use 7.0 as the default if MONGO_VERSION is not set environment: - # provide your credentials here - MONGO_INITDB_ROOT_USERNAME=travis - MONGO_INITDB_ROOT_PASSWORD=travis - MONGO_INITDB_DATABASE=ee2 - ports: ["27018:27017","27017:27017"] + ports: + - "27018:27017" + - "27017:27017" volumes: - - "./test/dockerfiles/mongo/docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/" + - ${INIT_PATH:-./test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/}:/docker-entrypoint-initdb.d/ diff --git a/kbase.yml b/kbase.yml index a7d366a81..6677b0823 100644 --- a/kbase.yml +++ b/kbase.yml @@ -8,7 +8,7 @@ service-language: python module-version: - 0.0.13 + 0.0.14 owners: [bsadkhin, tgu2, wjriehl, gaprice] diff --git a/lib/biokbase/log.py b/lib/biokbase/log.py index 5626ac03f..abe9d2077 100644 --- a/lib/biokbase/log.py +++ b/lib/biokbase/log.py @@ -77,15 +77,15 @@ from configparser import ConfigParser as _ConfigParser import time -MLOG_ENV_FILE = 'MLOG_CONFIG_FILE' -_GLOBAL = 'global' -MLOG_LOG_LEVEL = 'mlog_log_level' -MLOG_API_URL = 'mlog_api_url' -MLOG_LOG_FILE = 'mlog_log_file' +MLOG_ENV_FILE = "MLOG_CONFIG_FILE" +_GLOBAL = "global" +MLOG_LOG_LEVEL = "mlog_log_level" +MLOG_API_URL = "mlog_api_url" +MLOG_LOG_FILE = "mlog_log_file" DEFAULT_LOG_LEVEL = 6 -#MSG_CHECK_COUNT = 100 -#MSG_CHECK_INTERVAL = 300 # 300s = 5min +# MSG_CHECK_COUNT = 100 +# MSG_CHECK_INTERVAL = 300 # 300s = 5min MSG_FACILITY = _syslog.LOG_LOCAL1 EMERG_FACILITY = _syslog.LOG_LOCAL0 @@ -99,22 +99,31 @@ DEBUG = 7 DEBUG2 = 8 DEBUG3 = 9 -_MLOG_TEXT_TO_LEVEL = {'EMERG': EMERG, - 'ALERT': ALERT, - 'CRIT': CRIT, - 'ERR': ERR, - 'WARNING': WARNING, - 'NOTICE': NOTICE, - 'INFO': INFO, - 'DEBUG': DEBUG, - 'DEBUG2': DEBUG2, - 'DEBUG3': DEBUG3, - } -_MLOG_TO_SYSLOG = [_syslog.LOG_EMERG, _syslog.LOG_ALERT, _syslog.LOG_CRIT, - _syslog.LOG_ERR, _syslog.LOG_WARNING, _syslog.LOG_NOTICE, - _syslog.LOG_INFO, _syslog.LOG_DEBUG, _syslog.LOG_DEBUG, - _syslog.LOG_DEBUG] -#ALLOWED_LOG_LEVELS = set(_MLOG_TEXT_TO_LEVEL.values()) +_MLOG_TEXT_TO_LEVEL = { + "EMERG": EMERG, + "ALERT": ALERT, + "CRIT": CRIT, + "ERR": ERR, + "WARNING": WARNING, + "NOTICE": NOTICE, + "INFO": INFO, + "DEBUG": DEBUG, + "DEBUG2": DEBUG2, + "DEBUG3": DEBUG3, +} +_MLOG_TO_SYSLOG = [ + _syslog.LOG_EMERG, + _syslog.LOG_ALERT, + _syslog.LOG_CRIT, + _syslog.LOG_ERR, + _syslog.LOG_WARNING, + _syslog.LOG_NOTICE, + _syslog.LOG_INFO, + _syslog.LOG_DEBUG, + _syslog.LOG_DEBUG, + _syslog.LOG_DEBUG, +] +# ALLOWED_LOG_LEVELS = set(_MLOG_TEXT_TO_LEVEL.values()) _MLOG_LEVEL_TO_TEXT = {} for k, v in _MLOG_TEXT_TO_LEVEL.items(): _MLOG_LEVEL_TO_TEXT[v] = k @@ -128,21 +137,30 @@ class log(object): This class contains the methods necessary for sending log messages. """ - def __init__(self, subsystem, constraints=None, config=None, logfile=None, - ip_address=False, authuser=False, module=False, - method=False, call_id=False, changecallback=None): + def __init__( + self, + subsystem, + constraints=None, + config=None, + logfile=None, + ip_address=False, + authuser=False, + module=False, + method=False, + call_id=False, + changecallback=None, + ): if not subsystem: raise ValueError("Subsystem must be supplied") self.user = _getpass.getuser() - self.parentfile = _os.path.abspath(_inspect.getfile( - _inspect.stack()[1][0])) + self.parentfile = _os.path.abspath(_inspect.getfile(_inspect.stack()[1][0])) self.ip_address = ip_address self.authuser = authuser self.module = module self.method = method self.call_id = call_id - noop = lambda: None + noop = lambda: None # noqa self._callback = changecallback or noop self._subsystem = str(subsystem) self._mlog_config_file = config @@ -171,11 +189,11 @@ def _get_time_since_start(self): return time_diff def get_log_level(self): - if(self._user_log_level != -1): + if self._user_log_level != -1: return self._user_log_level - elif(self._config_log_level != -1): + elif self._config_log_level != -1: return self._config_log_level - elif(self._api_log_level != -1): + elif self._api_log_level != -1: return self._api_log_level else: return DEFAULT_LOG_LEVEL @@ -205,37 +223,39 @@ def update_config(self): if MLOG_LOG_LEVEL in cfgitems: try: self._config_log_level = int(cfgitems[MLOG_LOG_LEVEL]) - except: + except Exception: _warnings.warn( - 'Cannot parse log level {} from file {} to int'.format( - cfgitems[MLOG_LOG_LEVEL], self._mlog_config_file) - + '. Keeping current log level.') + "Cannot parse log level {} from file {} to int".format( + cfgitems[MLOG_LOG_LEVEL], self._mlog_config_file + ) + + ". Keeping current log level." + ) if MLOG_API_URL in cfgitems: api_url = cfgitems[MLOG_API_URL] if MLOG_LOG_FILE in cfgitems: self._config_log_file = cfgitems[MLOG_LOG_FILE] elif self._mlog_config_file: - _warnings.warn('Cannot read config file ' + self._mlog_config_file) + _warnings.warn("Cannot read config file " + self._mlog_config_file) - if (api_url): + if api_url: subsystem_api_url = api_url + "/" + self._subsystem try: - data = _json.load(_urllib2.urlopen(subsystem_api_url, - timeout=5)) + data = _json.load(_urllib2.urlopen(subsystem_api_url, timeout=5)) except _urllib2.URLError as e: code_ = None - if hasattr(e, 'code'): - code_ = ' ' + str(e.code) + if hasattr(e, "code"): + code_ = " " + str(e.code) _warnings.warn( - 'Could not connect to mlog api server at ' + - '{}:{} {}. Using default log level {}.'.format( - subsystem_api_url, code_, str(e.reason), - str(DEFAULT_LOG_LEVEL))) + "Could not connect to mlog api server at " + + "{}:{} {}. Using default log level {}.".format( + subsystem_api_url, code_, str(e.reason), str(DEFAULT_LOG_LEVEL) + ) + ) else: max_matching_level = -1 - for constraint_set in data['log_levels']: - level = constraint_set['level'] - constraints = constraint_set['constraints'] + for constraint_set in data["log_levels"]: + level = constraint_set["level"] + constraints = constraint_set["constraints"] if level <= max_matching_level: continue @@ -243,23 +263,25 @@ def update_config(self): for constraint in constraints: if constraint not in self._log_constraints: matches = 0 - elif (self._log_constraints[constraint] != - constraints[constraint]): + elif ( + self._log_constraints[constraint] != constraints[constraint] + ): matches = 0 if matches == 1: max_matching_level = level self._api_log_level = max_matching_level - if ((self.get_log_level() != loglevel or - self.get_log_file() != logfile) and not self._init): + if ( + self.get_log_level() != loglevel or self.get_log_file() != logfile + ) and not self._init: self._callback() def _resolve_log_level(self, level): - if(level in _MLOG_TEXT_TO_LEVEL): + if level in _MLOG_TEXT_TO_LEVEL: level = _MLOG_TEXT_TO_LEVEL[level] - elif(level not in _MLOG_LEVEL_TO_TEXT): - raise ValueError('Illegal log level') + elif level not in _MLOG_LEVEL_TO_TEXT: + raise ValueError("Illegal log level") return level def set_log_level(self, level): @@ -280,33 +302,40 @@ def set_log_file(self, filename): def set_log_msg_check_count(self, count): count = int(count) if count < 0: - raise ValueError('Cannot check a negative number of messages') + raise ValueError("Cannot check a negative number of messages") self._recheck_api_msg = count def set_log_msg_check_interval(self, interval): interval = int(interval) if interval < 0: - raise ValueError('interval must be positive') + raise ValueError("interval must be positive") self._recheck_api_time = interval def clear_user_log_level(self): self._user_log_level = -1 self._callback() - def _get_ident(self, level, user, parentfile, ip_address, authuser, module, - method, call_id): - infos = [self._subsystem, _MLOG_LEVEL_TO_TEXT[level], - repr(time.time()), user, parentfile, str(_os.getpid())] + def _get_ident( + self, level, user, parentfile, ip_address, authuser, module, method, call_id + ): + infos = [ + self._subsystem, + _MLOG_LEVEL_TO_TEXT[level], + repr(time.time()), + user, + parentfile, + str(_os.getpid()), + ] if self.ip_address: - infos.append(str(ip_address) if ip_address else '-') + infos.append(str(ip_address) if ip_address else "-") if self.authuser: - infos.append(str(authuser) if authuser else '-') + infos.append(str(authuser) if authuser else "-") if self.module: - infos.append(str(module) if module else '-') + infos.append(str(module) if module else "-") if self.method: - infos.append(str(method) if method else '-') + infos.append(str(method) if method else "-") if self.call_id: - infos.append(str(call_id) if call_id else '-') + infos.append(str(call_id) if call_id else "-") return "[" + "] [".join(infos) + "]" def _syslog(self, facility, level, ident, message): @@ -322,47 +351,75 @@ def _syslog(self, facility, level, ident, message): _syslog.closelog() def _log(self, ident, message): - ident = ' '.join([str(time.strftime( - "%Y-%m-%d %H:%M:%S", time.localtime())), - _platform.node(), ident + ': ']) + ident = " ".join( + [ + str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())), + _platform.node(), + ident + ": ", + ] + ) try: - with open(self.get_log_file(), 'a') as log: + with open(self.get_log_file(), "a") as log: if isinstance(message, str): - log.write(ident + message + '\n') + log.write(ident + message + "\n") else: try: for m in message: - log.write(ident + m + '\n') + log.write(ident + m + "\n") except TypeError: - log.write(ident + str(message) + '\n') + log.write(ident + str(message) + "\n") except Exception as e: - err = 'Could not write to log file ' + str(self.get_log_file()) + \ - ': ' + str(e) + '.' + err = ( + "Could not write to log file " + + str(self.get_log_file()) + + ": " + + str(e) + + "." + ) _warnings.warn(err) - def log_message(self, level, message, ip_address=None, authuser=None, - module=None, method=None, call_id=None): -# message = str(message) + def log_message( + self, + level, + message, + ip_address=None, + authuser=None, + module=None, + method=None, + call_id=None, + ): + # message = str(message) level = self._resolve_log_level(level) self.msg_count += 1 self._msgs_since_config_update += 1 - if(self._msgs_since_config_update >= self._recheck_api_msg - or self._get_time_since_start() >= self._recheck_api_time): + if ( + self._msgs_since_config_update >= self._recheck_api_msg + or self._get_time_since_start() >= self._recheck_api_time + ): self.update_config() - ident = self._get_ident(level, self.user, self.parentfile, ip_address, - authuser, module, method, call_id) + ident = self._get_ident( + level, + self.user, + self.parentfile, + ip_address, + authuser, + module, + method, + call_id, + ) # If this message is an emergency, send a copy to the emergency # facility first. - if(level == 0): + if level == 0: self._syslog(EMERG_FACILITY, level, ident, message) - if(level <= self.get_log_level()): + if level <= self.get_log_level(): self._syslog(MSG_FACILITY, level, ident, message) if self.get_log_file(): self._log(ident, message) -if __name__ == '__main__': + +if __name__ == "__main__": pass diff --git a/lib/execution_engine2/authclient.py b/lib/execution_engine2/authclient.py index 844f9b0c2..3602d257d 100644 --- a/lib/execution_engine2/authclient.py +++ b/lib/execution_engine2/authclient.py @@ -1,10 +1,10 @@ -''' +""" Created on Aug 1, 2016 A very basic KBase auth client for the Python server. @author: gaprice@lbl.gov -''' +""" import time as _time import requests as _requests import threading as _threading @@ -12,7 +12,7 @@ class TokenCache(object): - ''' A basic cache for tokens. ''' + """A basic cache for tokens.""" _MAX_TIME_SEC = 5 * 60 # 5 min @@ -24,7 +24,7 @@ def __init__(self, maxsize=2000): self._halfmax = maxsize / 2 # int division to round down def get_user(self, token): - token = hashlib.sha256(token.encode('utf-8')).hexdigest() + token = hashlib.sha256(token.encode("utf-8")).hexdigest() with self._lock: usertime = self._cache.get(token) if not usertime: @@ -37,16 +37,15 @@ def get_user(self, token): def add_valid_token(self, token, user): if not token: - raise ValueError('Must supply token') + raise ValueError("Must supply token") if not user: - raise ValueError('Must supply user') - token = hashlib.sha256(token.encode('utf-8')).hexdigest() + raise ValueError("Must supply user") + token = hashlib.sha256(token.encode("utf-8")).hexdigest() with self._lock: self._cache[token] = [user, _time.time()] if len(self._cache) > self._maxsize: sorted_items = sorted( - list(self._cache.items()), - key=(lambda v: v[1][1]) + list(self._cache.items()), key=(lambda v: v[1][1]) ) for i, (t, _) in enumerate(sorted_items): if i <= self._halfmax: @@ -56,16 +55,16 @@ def add_valid_token(self, token, user): class KBaseAuth(object): - ''' + """ A very basic KBase auth client for the Python server. - ''' + """ - _LOGIN_URL = 'https://kbase.us/services/auth/api/legacy/KBase/Sessions/Login' + _LOGIN_URL = "https://kbase.us/services/auth/api/legacy/KBase/Sessions/Login" def __init__(self, auth_url=None): - ''' + """ Constructor - ''' + """ self._authurl = auth_url if not self._authurl: self._authurl = self._LOGIN_URL @@ -73,22 +72,24 @@ def __init__(self, auth_url=None): def get_user(self, token): if not token: - raise ValueError('Must supply token') + raise ValueError("Must supply token") user = self._cache.get_user(token) if user: return user - d = {'token': token, 'fields': 'user_id'} + d = {"token": token, "fields": "user_id"} ret = _requests.post(self._authurl, data=d) if not ret.ok: try: err = ret.json() except Exception as e: ret.raise_for_status() - raise ValueError('Error connecting to auth service: {} {}\n{}' - .format(ret.status_code, ret.reason, - err['error']['message'])) + raise ValueError( + "Error connecting to auth service: {} {}\n{}".format( + ret.status_code, ret.reason, err["error"]["message"] + ) + ) - user = ret.json()['user_id'] + user = ret.json()["user_id"] self._cache.add_valid_token(token, user) return user diff --git a/lib/execution_engine2/sdk/EE2Runjob.py b/lib/execution_engine2/sdk/EE2Runjob.py index 5a7a8a635..745788eab 100644 --- a/lib/execution_engine2/sdk/EE2Runjob.py +++ b/lib/execution_engine2/sdk/EE2Runjob.py @@ -483,10 +483,10 @@ def run_batch( :return: A list of condor job ids or a failure notification """ - if type(params) != list: + if not isinstance(params, list): raise IncorrectParamsException("params must be a list") - if type(batch_params) != dict: + if not isinstance(batch_params, dict): raise IncorrectParamsException("batch params must be a mapping") wsid = batch_params.get(_WORKSPACE_ID) @@ -586,7 +586,7 @@ def _check_job_requirements_vs_admin( def _check_is_string(self, putative_str, name): if not putative_str: return None - if type(putative_str) != str: + if not isinstance(putative_str, str): raise IncorrectParamsException(f"{name} must be a string") return putative_str @@ -924,7 +924,7 @@ def run( """ # TODO Test this - if type(params) != dict: + if not isinstance(params, dict): raise IncorrectParamsException("params must be a mapping") self._preflight(runjob_params=params, as_admin=as_admin) @@ -949,10 +949,10 @@ def _get_job_reqs_from_concierge_params( rl = concierge_params.get(_REQUIREMENTS_LIST) schd_reqs = {} if rl: - if type(rl) != list: + if not isinstance(rl, list): raise IncorrectParamsException(f"{_REQUIREMENTS_LIST} must be a list") for s in rl: - if type(s) != str or "=" not in s: + if not isinstance(s, str) or "=" not in s: raise IncorrectParamsException( f"Found illegal requirement in {_REQUIREMENTS_LIST}: {s}" ) diff --git a/lib/execution_engine2/sdk/job_submission_parameters.py b/lib/execution_engine2/sdk/job_submission_parameters.py index 90bc58d73..6408f3108 100644 --- a/lib/execution_engine2/sdk/job_submission_parameters.py +++ b/lib/execution_engine2/sdk/job_submission_parameters.py @@ -124,9 +124,11 @@ def check_parameters( client_group, None if client_group_regex is None else bool(client_group_regex), _check_string(bill_to_user, "bill_to_user", optional=True), - None - if ignore_concurrency_limits is None - else bool(ignore_concurrency_limits), + ( + None + if ignore_concurrency_limits is None + else bool(ignore_concurrency_limits) + ), cls._check_scheduler_requirements(scheduler_requirements), None if debug_mode is None else bool(debug_mode), ) @@ -145,7 +147,7 @@ def _params(self): ) def __eq__(self, other): - if type(self) == type(other): + if type(self) == type(other): # noqa E721 return self._params() == ( other.cpus, other.memory_MB, @@ -214,7 +216,7 @@ def __init__( ) self.wsid = _gt_zero(wsid, "wsid", optional=True) source_ws_objects = source_ws_objects if source_ws_objects else [] - if type(source_ws_objects) != list: + if not isinstance(source_ws_objects, list): raise IncorrectParamsException("source_ws_objects must be a list") for i, ref in enumerate(source_ws_objects): upa, is_valid = _is_valid_UPA(ref) @@ -238,7 +240,7 @@ def _params(self): ) def __eq__(self, other): - if type(self) == type(other): + if type(self) == type(other): # noqa E721 return self._params() == ( other.job_id, other.app_info, diff --git a/lib/execution_engine2/utils/application_info.py b/lib/execution_engine2/utils/application_info.py index 5ebc6d02d..ce81e41c5 100644 --- a/lib/execution_engine2/utils/application_info.py +++ b/lib/execution_engine2/utils/application_info.py @@ -116,7 +116,7 @@ def get_application_id(self) -> str: return self.application_module def __eq__(self, other): - if type(self) == type(other): + if type(self) == type(other): # noqa E721 return ( self.module, self.method, diff --git a/lib/execution_engine2/utils/job_requirements_resolver.py b/lib/execution_engine2/utils/job_requirements_resolver.py index c0f17cdc6..92c753088 100644 --- a/lib/execution_engine2/utils/job_requirements_resolver.py +++ b/lib/execution_engine2/utils/job_requirements_resolver.py @@ -77,13 +77,13 @@ def _check_clientgroup(clientgroup, source): def _string_request(putative_string, name, source): - if type(putative_string) != str: + if not isinstance(putative_string, str): _check_raise(name, putative_string, source) return putative_string.strip() def _int_request(putative_int, original, name, source): - if type(putative_int) == float: + if isinstance(putative_int, float): _check_raise(f"{name} request", original, source) try: return int(putative_int) @@ -96,7 +96,7 @@ def _check_cpus(cpus, source): def _check_memory(memory, source): - if type(memory) == int: + if isinstance(memory, int): return memory memory2 = _string_request(memory, "memory request", source) if memory2.endswith("M"): @@ -107,7 +107,7 @@ def _check_memory(memory, source): def _check_disk(disk, source): - if type(disk) == int: + if isinstance(disk, int): return disk disk2 = _string_request(disk, "disk request", source) if disk2.endswith("GB"): @@ -116,7 +116,7 @@ def _check_disk(disk, source): def _bool_request(putative_bool, name, source): - if type(putative_bool) == bool or type(putative_bool) == int: + if isinstance(putative_bool, bool) or isinstance(putative_bool, int): return bool(putative_bool) pbs = _string_request(putative_bool, name, source).lower() if pbs == "true": @@ -347,7 +347,7 @@ def normalize_job_reqs( def _has_value(cls, inc): if inc is None: return False - if type(inc) == str and not inc.strip(): + if isinstance(inc, str) and not inc.strip(): return False return True diff --git a/lib/execution_engine2/utils/user_info.py b/lib/execution_engine2/utils/user_info.py index 1a7a4e10f..d1a2cbfba 100644 --- a/lib/execution_engine2/utils/user_info.py +++ b/lib/execution_engine2/utils/user_info.py @@ -28,7 +28,7 @@ def __init__(self, username: str, token: str): self.token = _check_string(token, "token") def __eq__(self, other): - if type(self) == type(other): + if type(self) == type(other): # noqa E721 return (self.username, self.token) == (other.username, other.token) return False diff --git a/lib/installed_clients/authclient.py b/lib/installed_clients/authclient.py index fa0a9ce01..3602d257d 100644 --- a/lib/installed_clients/authclient.py +++ b/lib/installed_clients/authclient.py @@ -12,7 +12,7 @@ class TokenCache(object): - """ A basic cache for tokens. """ + """A basic cache for tokens.""" _MAX_TIME_SEC = 5 * 60 # 5 min diff --git a/requirements.txt b/requirements.txt index be905f7b1..c5b928b43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,17 +7,16 @@ configparser==5.0.2 confluent-kafka==1.9.2 coverage==5.5 docker==5.0.0 -gevent==21.12.0 -greenlet==1.1.0 +gevent==24.2.1 ; python_version >= "3.10" and python_version < "4.0" gunicorn==20.1.0 -htcondor==9.12.0 +htcondor==23.8.1 Jinja2==3.0.1 JSONRPCBase==0.2.0 mock==4.0.3 maps==5.1.1 -mongoengine==0.23.1 +mongoengine==0.28.2 psutil==5.8.0 -pymongo==3.12.0 +pymongo==4.8.0 pytest==6.2.4 pytest-cov==2.12.1 python-dateutil==2.8.2 diff --git a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js b/test/dockerfiles/mongo/docker-entrypoint-initdb.d-3.6/mongo-init.js similarity index 82% rename from test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js rename to test/dockerfiles/mongo/docker-entrypoint-initdb.d-3.6/mongo-init.js index 9a9ec8bf6..c9e30a380 100755 --- a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js +++ b/test/dockerfiles/mongo/docker-entrypoint-initdb.d-3.6/mongo-init.js @@ -1,7 +1,6 @@ +print("Adding travis username to ee2 database for mongo 3.6") db.auth('travis', 'travis') - db = db.getSiblingDB('ee2') - db.createUser( { user: "travis", diff --git a/test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/mongo-init.js b/test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/mongo-init.js new file mode 100755 index 000000000..729808929 --- /dev/null +++ b/test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/mongo-init.js @@ -0,0 +1,14 @@ +print("Adding travis username to ee2 database for mongo 7.0") +db = db.getSiblingDB('ee2') +db.createUser( + { + user: "travis", + pwd: "travis", + roles: [ + { + role: "dbOwner", + db: "ee2" + } + ] + } +); \ No newline at end of file diff --git a/test/tests_for_auth/ee2_admin_mode_test.py b/test/tests_for_auth/ee2_admin_mode_test.py index 15ee0682a..b12c30e06 100644 --- a/test/tests_for_auth/ee2_admin_mode_test.py +++ b/test/tests_for_auth/ee2_admin_mode_test.py @@ -287,12 +287,14 @@ def test_no_user(self): runner = self.getRunner() method_1 = "module_name.function_name" job_params_1 = get_sample_job_params(method=method_1, wsid=self.ws_id) - + error_regex = ( + r"\('An error occurred while fetching user permissions from the Workspace', " + r"ServerError\('Token validation failed: Auth service returned an error: 10020 Invalid token'\)\)" + ) with self.assertRaisesRegex( - expected_exception=RuntimeError, - expected_regex=r"ServerError\('Token validation failed: Login failed! Server responded with code 401 Unauthorized'\)", + expected_exception=RuntimeError, expected_regex=error_regex ): - runner.run_job(params=job_params_1, as_admin=False) + runner.run_job(params=job_params_1) def test_admin_reader(self): # Admin User with READ