diff --git a/Dockerfile b/Dockerfile index 9a66b697..ed923db2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ MAINTAINER KBase Developer RUN pip install --upgrade --extra-index-url=https://pypi.anaconda.org/kbase/simple \ pip \ biopython==1.70 \ - releng-client==0.0.1 + releng-client==0.0.1 \ + pytest==7.0.1 COPY ./ /kb/module RUN mkdir -p /kb/module/work diff --git a/lib/GenomeFileUtil/GenomeFileUtilImpl.py b/lib/GenomeFileUtil/GenomeFileUtilImpl.py index 5f7ad21d..c4b84c11 100644 --- a/lib/GenomeFileUtil/GenomeFileUtilImpl.py +++ b/lib/GenomeFileUtil/GenomeFileUtilImpl.py @@ -10,7 +10,11 @@ from GenomeFileUtil.core.FastaGFFToGenome import FastaGFFToGenome from GenomeFileUtil.core.GenbankToGenome import GenbankToGenome from GenomeFileUtil.core.GenomeFeaturesToFasta import GenomeFeaturesToFasta -from GenomeFileUtil.core.GenomeInterface import GenomeInterface +from GenomeFileUtil.core.GenomeInterface import ( + GenomeInterface, + MAX_THREADS_DEFAULT, + THREADS_PER_CPU_DEFAULT, +) from GenomeFileUtil.core.GenomeToGFF import GenomeToGFF from GenomeFileUtil.core.GenomeToGenbank import GenomeToGenbank from installed_clients.AssemblyUtilClient import AssemblyUtil @@ -32,6 +36,18 @@ def __init__(self, config): self.re_api_url = config['re-api-url'] self.raw = config + +def _validate_catalog_param_type(threads_count, var_name, default_val, expected_type): + """Helper function to validate catalog params type""" + if threads_count is None: + print(f"Cannot retrieve {var_name} from the catalog, set {var_name}={default_val}") + return default_val + print(f"Successfully retrieved {var_name} from the catalog!") + try: + threads_count = expected_type(threads_count) + except ValueError as e: + raise ValueError(f"{var_name} must be of type {expected_type.__name__}") from e + return threads_count #END_HEADER @@ -64,6 +80,15 @@ def __init__(self, config): self.cfg = SDKConfig(config) logging.basicConfig(format='%(created)s %(levelname)s: %(message)s', level=logging.INFO) + + max_threads = os.environ.get("KBASE_SECURE_CONFIG_PARAM_MAX_THREADS") + threads_per_cpu = os.environ.get("KBASE_SECURE_CONFIG_PARAM_THREADS_PER_CPU") + self.max_threads = _validate_catalog_param_type( + max_threads, "MAX_THREADS", MAX_THREADS_DEFAULT, int + ) + self.threads_per_cpu = _validate_catalog_param_type( + threads_per_cpu, "THREADS_PER_CPU", THREADS_PER_CPU_DEFAULT, float + ) #END_CONSTRUCTOR pass diff --git a/lib/GenomeFileUtil/core/GenomeInterface.py b/lib/GenomeFileUtil/core/GenomeInterface.py index def9a499..b1aaa3ce 100644 --- a/lib/GenomeFileUtil/core/GenomeInterface.py +++ b/lib/GenomeFileUtil/core/GenomeInterface.py @@ -16,6 +16,10 @@ MAX_GENOME_SIZE = 2**30 +# catalog params +MAX_THREADS_DEFAULT = 10 +THREADS_PER_CPU_DEFAULT = 1 + class GenomeInterface: def __init__(self, config): diff --git a/scripts/run_tests_within_container.sh b/scripts/run_tests_within_container.sh index 5ce658c0..b6c86a61 100644 --- a/scripts/run_tests_within_container.sh +++ b/scripts/run_tests_within_container.sh @@ -2,7 +2,8 @@ script_dir=$(dirname "$(readlink -f "$0")") export KB_DEPLOYMENT_CONFIG=$script_dir/../deploy.cfg export KB_AUTH_TOKEN=`cat /kb/module/work/token` -export PYTHONPATH=$script_dir/../lib:$PATH:$PYTHONPATH +export PYTHONPATH=$script_dir/../lib:$PYTHONPATH +export PYTHONPATH=$script_dir/../test:$PYTHONPATH # Set TEST_PATH to run a specific test. Eg: TEST_PATH=test.core.update_taxon_assignments_test export TEST_PATH=. diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..fd8d9123 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,7 @@ +import traceback + + +def assert_exception_correct(got: Exception, expected: Exception): + err = "".join(traceback.TracebackException.from_exception(got).format()) + assert got.args == expected.args, err + assert type(got) == type(expected) diff --git a/test/genome_tests/__init__.py b/test/genome_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/genome_tests/impl_test.py b/test/genome_tests/impl_test.py new file mode 100644 index 00000000..a40dd8d6 --- /dev/null +++ b/test/genome_tests/impl_test.py @@ -0,0 +1,52 @@ +import os +import pytest +import unittest + +from GenomeFileUtil.GenomeFileUtilImpl import GenomeFileUtil +from conftest import assert_exception_correct + + +class ImplTest(unittest.TestCase): + def setUp(self): + self.cfg = {} + self.cfg["workspace-url"] = "https://ci.kbase.us/services/ws" + self.cfg["shock-url"] = "https://ci.kbase.us/services/shock-api" + self.cfg["handle-service-url"] = "https://ci.kbase.us/services/handle_service" + self.cfg["scratch"] = "/kb/module/work/tmp" + self.cfg["srv-wiz-url"] = "https://ci.kbase.us/services/service_wizard" + self.cfg["auth-service-url"] = "https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login" + self.cfg["re-api-url"] = "https://ci.kbase.us/services/relation_engine_api" + + def tearDown(self): + for kbase_secure_param in ( + "KBASE_SECURE_CONFIG_PARAM_MAX_THREADS", + "KBASE_SECURE_CONFIG_PARAM_THREADS_PER_CPU", + ): + os.environ.pop(kbase_secure_param, None) + + def _run_test_fail(self, cfg, error_message): + with pytest.raises(Exception) as got: + GenomeFileUtil(cfg) + assert_exception_correct(got.value, ValueError(error_message)) + + def test_valid_catalog_param_type(self): + os.environ["KBASE_SECURE_CONFIG_PARAM_MAX_THREADS"] = "16" + os.environ["KBASE_SECURE_CONFIG_PARAM_THREADS_PER_CPU"] = "2.5" + gfu = GenomeFileUtil(self.cfg) + self.assertEqual(gfu.max_threads, 16) + self.assertEqual(gfu.threads_per_cpu, 2.5) + + def test_valid_default_catalog_param_type(self): + gfu = GenomeFileUtil(self.cfg) + self.assertEqual(gfu.max_threads, 10) + self.assertEqual(gfu.threads_per_cpu, 1) + + def test_invalid_max_threads(self): + os.environ["KBASE_SECURE_CONFIG_PARAM_MAX_THREADS"] = "10.5" + os.environ["KBASE_SECURE_CONFIG_PARAM_THREADS_PER_CPU"] = "2.5" + self._run_test_fail(self.cfg, "MAX_THREADS must be of type int") + + def test_invalid_threads_per_cpu(self): + os.environ["KBASE_SECURE_CONFIG_PARAM_MAX_THREADS"] = "10" + os.environ["KBASE_SECURE_CONFIG_PARAM_THREADS_PER_CPU"] = "2.8e" + self._run_test_fail(self.cfg, "THREADS_PER_CPU must be of type float")