diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 6a10df7a0..cce45ce9d 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -63,14 +63,6 @@ jobs: run: | python3 -c 'import numpy as np; print(np.get_include())' - - name: Workaround - install CMake 3.25.2 since 3.26.0 doesn't work - # Normally we would install cmake with the package manager, but - # ubuntu 20.04 doesn't seems to keep older versions of cmake around... - # Fortunately, there exists a pip package - run: | - python3 -m pip install cmake==3.25.2 - cmake --version - - name: List installed Python packages run: | uname -a @@ -78,6 +70,8 @@ jobs: pip freeze - name: Configure + env: + DLITE_IMPORTSKIP_EXITCODE: 1 run: | Python3_ROOT=$(python3 -c 'import sys; print(sys.exec_prefix)') \ CFLAGS='-Wno-missing-field-initializers' \ diff --git a/CMakeLists.txt b/CMakeLists.txt index f76d27bac..97cd40abc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -336,6 +336,13 @@ if(WITH_PYTHON) # endif() # message(STATUS "ENV{Python3_LIBRARY}: $ENV{Python3_LIBRARY}") + add_custom_target(wheel + COMMAND ${Python3_EXECUTABLE} -m pip wheel -w dist + ${dlite_SOURCE_DIR}/python + DEPENDS ${dlite} + COMMENT "Build Python wheels" + ) + message(STATUS "CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}") message(STATUS "Python3_prefix = ${Python3_prefix}") @@ -942,6 +949,7 @@ configure_package_config_file( PATH_VARS ${DLITE_CONFIG_VARS} ) + ################################################################# # Install diff --git a/bindings/python/tests/CMakeLists.txt b/bindings/python/tests/CMakeLists.txt index a5d0a7e2a..d72b873ee 100644 --- a/bindings/python/tests/CMakeLists.txt +++ b/bindings/python/tests/CMakeLists.txt @@ -54,6 +54,8 @@ foreach(test ${tests}) ENVIRONMENT "DLITE_USE_BUILD_ROOT=YES") # Skip tests that exit with return code 44 + set_property(TEST ${name} APPEND PROPERTY + ENVIRONMENT "DLITE_IMPORTSKIP_EXITCODE=$ENV{DLITE_IMPORTSKIP_EXITCODE}") set_property(TEST ${name} PROPERTY SKIP_RETURN_CODE 44) endforeach() diff --git a/bindings/python/testutils.py b/bindings/python/testutils.py index 4cc6fefa3..1e356e3a5 100644 --- a/bindings/python/testutils.py +++ b/bindings/python/testutils.py @@ -1,5 +1,6 @@ """Some utilities for testing.""" import importlib +import os import socket import sys @@ -71,15 +72,44 @@ def importcheck(module_name, package=None): return None -def importskip(module_name, package=None, exitcode=44): +def importskip(module_name, package=None, exitcode=44, + env_exitcode="DLITE_IMPORTSKIP_EXITCODE"): """Import and return the requested module. Calls `sys.exit()` with given exitcode if the module cannot be imported. + + Arguments: + module_name: Name of module to try to import. + package: Optional package name that the module might reside in. + exitcode: The default exit code of `sys.exit()` if the module cannot + be imported. + env_exitcode: Name of environment variable containing an exitcode + to call `sys.exit()` with if the module cannot be imported. + This overrides `exitcode`. + + Notes: + If you want to run the tests and get an error if a module + cannot be imported, set environment variable + `DLITE_IMPORTSKIP_EXITCODE=1` before running the tests (if you + run with ctest, set `DLITE_IMPORTSKIP_EXITCODE` at configure + time). + + For packages that depend on external services like postgresql, + call `importskip()` with `env_exitcode=None` to skip the test + regardless of whether `DLITE_IMPORTSKIP_EXITCODE` is set or not. + """ try: return importlib.import_module(module_name, package=package) except ModuleNotFoundError as exc: - print(f"{exc}: skipping test", file=sys.stderr) + if env_exitcode and env_exitcode in os.environ: + try: + exitcode = int(os.environ[env_exitcode]) + except: + pass + else: + print(f"{exc}: skipping test", file=sys.stderr) + sys.exit(exitcode) diff --git a/doc/user_guide/environment_variables.md b/doc/user_guide/environment_variables.md index 8ed012d9b..3c1584a02 100644 --- a/doc/user_guide/environment_variables.md +++ b/doc/user_guide/environment_variables.md @@ -96,6 +96,11 @@ DLITE_USE_BUILD_ROOT is set). - a glob pattern (/path/to/*.json) In the two last cases, the file extension must match the driver name. + - **DLITE_IMPORTSKIP_EXITCODE**: Exit code from tests that fail to + load a python module with `importskip()`. Define this to 1 at + configure time if you want the tests not to be skipped if they + cannot import a needed module. + Environment variables for controlling error handling ---------------------------------------------------- diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index b13d48b6d..e032d4ddc 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -107,6 +107,9 @@ if(FORCE_EXAMPLES OR EXISTS ${CMAKE_INSTALL_PREFIX}/share/dlite/examples) endif() set_property(TEST ${name} APPEND PROPERTY ENVIRONMENT "DLITE_USE_BUILD_ROOT=YES") + set_property(TEST ${name} APPEND PROPERTY + ENVIRONMENT "DLITE_IMPORTSKIP_EXITCODE=$ENV{DLITE_IMPORTSKIP_EXITCODE}") + set_property(TEST ${name} PROPERTY SKIP_RETURN_CODE 44) endforeach() diff --git a/examples/mappings/oteexample.py b/examples/mappings/oteexample.py index ec08857b4..c75987dfa 100644 --- a/examples/mappings/oteexample.py +++ b/examples/mappings/oteexample.py @@ -1,16 +1,17 @@ """Mapping example using OTELib.""" from pathlib import Path -try: - from tripper import EMMO, MAP, Triplestore - from otelib import OTEClient - import oteapi_dlite # To check that it is installed -except ModuleNotFoundError as exc: - print(f"Skipping OTE example because of missing module: {exc}") - import sys - sys.exit(44) # Exit code 44 -> skip test because of missing dependencies - import dlite +from dlite.testutils import importskip + +importskip("tripper") +from tripper import EMMO, MAP, Triplestore + +importskip("otelib", env_exitcode=None) +from otelib import OTEClient + +oteapi_dlite = importskip("oteapi_dlite", env_exitcode=None) + # Paths diff --git a/examples/read-csv/main.py b/examples/read-csv/main.py index c7961a81f..cbebd35a9 100644 --- a/examples/read-csv/main.py +++ b/examples/read-csv/main.py @@ -1,8 +1 @@ -try: - import pandas # noqa: F401 - import tables # noqa: F401 -except ImportError: - import sys - sys.exit(44) # skip this test if pandas is not available - import readcsv # noqa: F401 diff --git a/examples/read-csv/readcsv.py b/examples/read-csv/readcsv.py index 8b1a9e545..1ad5cd223 100644 --- a/examples/read-csv/readcsv.py +++ b/examples/read-csv/readcsv.py @@ -1,6 +1,11 @@ from pathlib import Path import dlite +from dlite.testutils import importskip + +importskip("pandas") +importskip("tables") +importskip("yaml") # Set up some paths diff --git a/storages/python/tests-python/CMakeLists.txt b/storages/python/tests-python/CMakeLists.txt index 7cc78d8be..862cfff8d 100644 --- a/storages/python/tests-python/CMakeLists.txt +++ b/storages/python/tests-python/CMakeLists.txt @@ -78,6 +78,8 @@ foreach(test ${python-tests}) set_property(TEST ${test} APPEND PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${dlite_LD_LIBRARY_PATH_NATIVE}") endif() + set_property(TEST ${name} APPEND PROPERTY + ENVIRONMENT "DLITE_IMPORTSKIP_EXITCODE=$ENV{DLITE_IMPORTSKIP_EXITCODE}") # Skip tests that exit with return code 44 set_property(TEST ${test} PROPERTY SKIP_RETURN_CODE 44) diff --git a/storages/python/tests-python/test_image.py b/storages/python/tests-python/test_image.py index a27bf7922..0092c7fbb 100644 --- a/storages/python/tests-python/test_image.py +++ b/storages/python/tests-python/test_image.py @@ -2,12 +2,12 @@ from pathlib import Path import numpy as np + import dlite +from dlite.testutils import importskip -try: - import skimage -except ImportError: - sys.exit(44) # skip test +importskip("scipy") +importskip("skimage") thisdir = Path(__file__).absolute().parent diff --git a/storages/python/tests-python/test_mongodb-atlas_python.py b/storages/python/tests-python/test_mongodb-atlas_python.py index b3b639db4..621b7b9d5 100644 --- a/storages/python/tests-python/test_mongodb-atlas_python.py +++ b/storages/python/tests-python/test_mongodb-atlas_python.py @@ -1,7 +1,12 @@ import sys import os from pathlib import Path + import dlite +from dlite.testutils import importskip + +importskip("pymongo", env_exitcode=None) + # Get the current file path current_file = Path(__file__).resolve() diff --git a/storages/python/tests-python/test_mongodb_python.py b/storages/python/tests-python/test_mongodb_python.py index 723e131ee..064d9f06f 100644 --- a/storages/python/tests-python/test_mongodb_python.py +++ b/storages/python/tests-python/test_mongodb_python.py @@ -7,8 +7,10 @@ from dlite.options import Options from dlite.testutils import importskip -importskip("pymongo") -mongomock = importskip("mongomock") +importskip("pymongo", env_exitcode=None) +mongomock = importskip("mongomock", env_exitcode=None) + +from dlite.testutils import importskip @mongomock.patch(servers=(('localhost', 27017),)) diff --git a/storages/python/tests-python/test_postgresql_storage_python.py b/storages/python/tests-python/test_postgresql_storage_python.py index 83be9530c..61989974f 100644 --- a/storages/python/tests-python/test_postgresql_storage_python.py +++ b/storages/python/tests-python/test_postgresql_storage_python.py @@ -5,14 +5,14 @@ from pathlib import Path sys.dont_write_bytecode = True -try: - import psycopg2 -except ImportError: - sys.exit(44) -from psycopg2 import sql import dlite from dlite.utils import instance_from_dict +from dlite.testutils import importskip + +psycopg2 = importskip("psycopg2", env_exitcode=None) +from psycopg2 import sql + from run_python_storage_tests import print_test_exception diff --git a/storages/python/tests-python/test_redis.py b/storages/python/tests-python/test_redis.py index e2ff6a706..662a3b826 100644 --- a/storages/python/tests-python/test_redis.py +++ b/storages/python/tests-python/test_redis.py @@ -7,7 +7,7 @@ import dlite from dlite.testutils import importskip, serverskip -importskip("redis") # skip this test if redis is not available +importskip("redis", env_exitcode=None) # skip this test if redis is not available serverskip("localhost", 6379) # skip test if redis is down