From ec5cabdbd6d510a016ca0f7ebe39221be3386495 Mon Sep 17 00:00:00 2001 From: Mikhail Beck Date: Wed, 29 Nov 2023 12:39:20 +0000 Subject: [PATCH] Made the container uploading and language registration two separate actions (#153) * Made the container uploading and language registration two separate actions * [CodeBuild] * Addressed review comments [CodeBuild] * More review comments [CodeBuild] * Some User Guide updates [CodeBuild] * Some User Guide updates [CodeBuild] --- doc/user_guide/user_guide.md | 32 ++++++- .../deployment/language_container_deployer.py | 96 ++++++++++++++----- .../language_container_deployer_cli.py | 44 ++++++++- .../test_language_container_deployer.py | 94 +++++++++++++++--- .../test_language_container_deployer_cli.py | 6 +- .../deployment/test_scripts_deployer_cli.py | 5 +- ...est_language_container_deployer_cli_run.py | 48 ++++++++++ 7 files changed, 271 insertions(+), 54 deletions(-) create mode 100644 tests/unit_tests/deployment/test_language_container_deployer_cli_run.py diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index 1b65a4a4..f34c6599 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -116,6 +116,29 @@ The option `--use-ssl-cert-validation`is the default, you can disable it with `- Use caution if you want to turn certificate validation off as it potentially lowers the security of your Database connection. +By default, the above command will upload and activate the language container at the System level. +The latter requires you to have the System Privileges, as it will attempt to change DB system settings. +If such privileges cannot be granted the activation can be skipped by using the `--no-alter-system` option. +The command will then print two possible language activation SQL queries, which look like the following: +```sql +ALTER SESSION SET SCRIPT_LANGUAGES=... +ALTER SYSTEM SET SCRIPT_LANGUAGES=... +``` +These queries represent two alternative ways of activating a language container. The first one activates the +container at the [Session level](https://docs.exasol.com/db/latest/sql/alter_session.htm). It doesn't require +System Privileges. However, it must be run every time a new session starts. The second one activates the container +at the [System level](https://docs.exasol.com/db/latest/sql/alter_system.htm). It needs to be run just once, +but it does require System Privileges. It may be executed by a database administrator. Please note, that changes +made at the system level only become effective in new sessions, as described +[here](https://docs.exasol.com/db/latest/sql/alter_system.htm#microcontent1). + +It is also possible to activate the language without repeatedly uploading the container. If the container +has already been uploaded one can use the `--no-upload-container` option to skip this step. + +By default, overriding language activation is not permitted. If a language with the same alias has already +been activated the command will result in an error. To override the activation, you can use the +`--allow-override` option. + #### Customized Installation In this installation, you can install the desired or customized language container. In the following steps, it is explained how to install the @@ -132,8 +155,8 @@ There are two ways to install the language container: (1) using a python script 1. *Installation with Python Script* To install the language container, it is necessary to load the container - into the BucketFS and register it to the database. The following command - provides this setup using the python script provided with this library: + into the BucketFS and activate it in the database. The following command + performs this setup using the python script provided with this library: ```buildoutcfg python -m exasol_transformers_extension.deploy language-container @@ -150,6 +173,9 @@ There are two ways to install the language container: (1) using a python script --language-alias \ --container-file ``` + Please note, that all considerations described in the Quick Installation + section are still applicable. + 2. *Manual Installation* @@ -171,7 +197,7 @@ There are two ways to install the language container: (1) using a python script ``` The uploaded container should be secondly activated through adjusting - the session parameter `SCRIPT_LANGUAGES`. The activation can be scoped + the session parameter `SCRIPT_LANGUAGES`. As it was mentioned before, the activation can be scoped either session-wide (`ALTER SESSION`) or system-wide (`ALTER SYSTEM`). The following example query activates the container session-wide: diff --git a/exasol_transformers_extension/deployment/language_container_deployer.py b/exasol_transformers_extension/deployment/language_container_deployer.py index 5fecd024..8da2a82a 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer.py +++ b/exasol_transformers_extension/deployment/language_container_deployer.py @@ -1,5 +1,6 @@ +from enum import Enum import pyexasol -from typing import List +from typing import List, Optional from pathlib import Path, PurePosixPath from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation import logging @@ -10,6 +11,15 @@ logger = logging.getLogger(__name__) +class LanguageActivationLevel(Enum): + f""" + Language activation level, i.e. + ALTER SET SCRIPT_LANGUAGES=... + """ + Session = 'SESSION' + System = 'SYSTEM' + + class LanguageContainerDeployer: def __init__(self, pyexasol_connection: pyexasol.ExaConnection, @@ -22,14 +32,21 @@ def __init__(self, self._pyexasol_conn = pyexasol_connection logger.debug(f"Init {LanguageContainerDeployer.__name__}") - def deploy_container(self): - path_in_udf = self._upload_container() - for alter in ["SESSION", "SYSTEM"]: - alter_command = self._generate_alter_command(alter, path_in_udf) - self._pyexasol_conn.execute(alter_command) - logging.debug(alter_command) + def deploy_container(self, allow_override: bool = False) -> None: + """ + Uploads the SLC and activates it at the SYSTEM level. + + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. + """ + path_in_udf = self.upload_container() + self.activate_container(LanguageActivationLevel.System, allow_override, path_in_udf) - def _upload_container(self) -> PurePosixPath: + def upload_container(self) -> PurePosixPath: + """ + Uploads the SLC. + Returns the path where the container is uploaded as it's seen by a UDF. + """ if not self._container_file.is_file(): raise RuntimeError(f"Container file {self._container_file} " f"is not a file.") @@ -40,20 +57,49 @@ def _upload_container(self) -> PurePosixPath: logging.debug("Container is uploaded to bucketfs") return PurePosixPath(path_in_udf) - def _generate_alter_command(self, alter_type: str, - path_in_udf: PurePosixPath) -> str: + def activate_container(self, alter_type: LanguageActivationLevel = LanguageActivationLevel.Session, + allow_override: bool = False, + path_in_udf: Optional[PurePosixPath] = None) -> None: + """ + Activates the SLC container at the required level. + + alter_type - Language activation level, defaults to the SESSION. + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. + path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF. + """ + alter_command = self.generate_activation_command(alter_type, allow_override, path_in_udf) + self._pyexasol_conn.execute(alter_command) + logging.debug(alter_command) + + def generate_activation_command(self, alter_type: LanguageActivationLevel, + allow_override: bool = False, + path_in_udf: Optional[PurePosixPath] = None) -> str: + """ + Generates an SQL command to activate the SLC container at the required level. The command will + preserve existing activations of other containers identified by different language aliases. + Activation of a container with the same alias, if exists, will be overwritten. + + alter_type - Activation level - SYSTEM or SESSION. + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. + path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF. + """ + if path_in_udf is None: + path_in_udf = self._bucketfs_location.generate_bucket_udf_path(self._container_file.name) new_settings = \ - self._update_previous_language_settings(alter_type, path_in_udf) + self._update_previous_language_settings(alter_type, allow_override, path_in_udf) alter_command = \ - f"ALTER {alter_type} SET SCRIPT_LANGUAGES='{new_settings}';" + f"ALTER {alter_type.value} SET SCRIPT_LANGUAGES='{new_settings}';" return alter_command - def _update_previous_language_settings(self, alter_type: str, + def _update_previous_language_settings(self, alter_type: LanguageActivationLevel, + allow_override: bool, path_in_udf: PurePosixPath) -> str: prev_lang_settings = self._get_previous_language_settings(alter_type) prev_lang_aliases = prev_lang_settings.split(" ") self._check_if_requested_language_alias_already_exists( - prev_lang_aliases) + allow_override, prev_lang_aliases) new_definitions_str = self._generate_new_language_settings( path_in_udf, prev_lang_aliases) return new_definitions_str @@ -73,26 +119,30 @@ def _generate_new_language_settings(self, path_in_udf: PurePosixPath, return new_definitions_str def _check_if_requested_language_alias_already_exists( - self, prev_lang_aliases: List[str]) -> None: + self, allow_override: bool, + prev_lang_aliases: List[str]) -> None: definition_for_requested_alias = [ alias_definition for alias_definition in prev_lang_aliases if alias_definition.startswith(self._language_alias + "=")] if not len(definition_for_requested_alias) == 0: - logging.warning(f"The requested language alias " - f"{self._language_alias} is already in use.") + warning_message = f"The requested language alias {self._language_alias} is already in use." + if allow_override: + logging.warning(warning_message) + else: + raise RuntimeError(warning_message) - def _get_previous_language_settings(self, alter_type: str) -> str: + def _get_previous_language_settings(self, alter_type: LanguageActivationLevel) -> str: result = self._pyexasol_conn.execute( - f"""SELECT "{alter_type}_VALUE" FROM SYS.EXA_PARAMETERS WHERE + f"""SELECT "{alter_type.value}_VALUE" FROM SYS.EXA_PARAMETERS WHERE PARAMETER_NAME='SCRIPT_LANGUAGES'""").fetchall() return result[0][0] @classmethod - def run(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, + def create(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, bucketfs_use_https: bool, bucketfs_user: str, container_file: Path, bucketfs_password: str, bucket: str, path_in_bucket: str, dsn: str, db_user: str, db_password: str, language_alias: str, - ssl_cert_path: str = None, use_ssl_cert_validation: bool = True): + ssl_cert_path: str = None, use_ssl_cert_validation: bool = True) -> "LanguageContainerDeployer": websocket_sslopt = get_websocket_ssl_options(use_ssl_cert_validation, ssl_cert_path) @@ -108,6 +158,4 @@ def run(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, bucketfs_name, bucketfs_host, bucketfs_port, bucketfs_use_https, bucketfs_user, bucketfs_password, bucket, path_in_bucket) - language_container_deployer = cls( - pyexasol_conn, language_alias, bucketfs_location, container_file) - language_container_deployer.deploy_container() + return cls(pyexasol_conn, language_alias, bucketfs_location, container_file) diff --git a/exasol_transformers_extension/deployment/language_container_deployer_cli.py b/exasol_transformers_extension/deployment/language_container_deployer_cli.py index b8500828..e8224c53 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer_cli.py +++ b/exasol_transformers_extension/deployment/language_container_deployer_cli.py @@ -1,9 +1,34 @@ import os import click from pathlib import Path +from textwrap import dedent from exasol_transformers_extension.deployment import deployment_utils as utils from exasol_transformers_extension.deployment.language_container_deployer import \ - LanguageContainerDeployer + LanguageContainerDeployer, LanguageActivationLevel + + +def run_deployer(deployer, upload_container: bool = True, + alter_system: bool = True, + allow_override: bool = False) -> None: + if upload_container and alter_system: + deployer.deploy_container(allow_override) + elif upload_container: + deployer.upload_container() + elif alter_system: + deployer.activate_container(LanguageActivationLevel.System, allow_override) + + if not alter_system: + message = dedent(f""" + In SQL, you can activate the SLC of the Transformers Extension + by using the following statements: + + To activate the SLC only for the current session: + {deployer.generate_activation_command(LanguageActivationLevel.Session, True)} + + To activate the SLC on the system: + {deployer.generate_activation_command(LanguageActivationLevel.System, True)} + """) + print(message) @click.command(name="language-container") @@ -28,6 +53,9 @@ @click.option('--language-alias', type=str, default="PYTHON3_TE") @click.option('--ssl-cert-path', type=str, default="") @click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True) +@click.option('--upload-container/--no-upload_container', type=bool, default=True) +@click.option('--alter-system/--no-alter-system', type=bool, default=True) +@click.option('--allow-override/--disallow-override', type=bool, default=False) def language_container_deployer_main( bucketfs_name: str, bucketfs_host: str, @@ -44,9 +72,13 @@ def language_container_deployer_main( db_pass: str, language_alias: str, ssl_cert_path: str, - use_ssl_cert_validation: bool): + use_ssl_cert_validation: bool, + upload_container: bool, + alter_system: bool, + allow_override: bool): + def call_runner(): - LanguageContainerDeployer.run( + deployer = LanguageContainerDeployer.create( bucketfs_name=bucketfs_name, bucketfs_host=bucketfs_host, bucketfs_port=bucketfs_port, @@ -61,8 +93,10 @@ def call_runner(): db_password=db_pass, language_alias=language_alias, ssl_cert_path=ssl_cert_path, - use_ssl_cert_validation=use_ssl_cert_validation - ) + use_ssl_cert_validation=use_ssl_cert_validation) + run_deployer(deployer, upload_container=upload_container, alter_system=alter_system, + allow_override=allow_override) + if container_file: call_runner() elif version: diff --git a/tests/integration_tests/with_db/deployment/test_language_container_deployer.py b/tests/integration_tests/with_db/deployment/test_language_container_deployer.py index 12c94247..21183a59 100644 --- a/tests/integration_tests/with_db/deployment/test_language_container_deployer.py +++ b/tests/integration_tests/with_db/deployment/test_language_container_deployer.py @@ -1,14 +1,18 @@ import textwrap +from typing import Callable from pathlib import Path +import pytest from _pytest.fixtures import FixtureRequest +from tests.fixtures.language_container_fixture import export_slc, flavor_path +from tests.fixtures.database_connection_fixture import pyexasol_connection from exasol_bucketfs_utils_python.bucketfs_factory import BucketFSFactory from exasol_script_languages_container_tool.lib.tasks.export.export_info import ExportInfo from pyexasol import ExaConnection from pytest_itde import config from exasol_transformers_extension.deployment.language_container_deployer \ - import LanguageContainerDeployer + import LanguageContainerDeployer, LanguageActivationLevel from tests.utils.parameters import bucketfs_params from tests.utils.revert_language_settings import revert_language_settings @@ -17,6 +21,8 @@ def test_language_container_deployer( request: FixtureRequest, export_slc: ExportInfo, pyexasol_connection: ExaConnection, + connection_factory: Callable[[config.Exasol], ExaConnection], + exasol_config: config.Exasol, bucketfs_config: config.BucketFs, ): test_name: str = request.node.name @@ -25,11 +31,69 @@ def test_language_container_deployer( container_path = Path(export_slc.cache_file) with revert_language_settings(pyexasol_connection): create_schema(pyexasol_connection, schema) - call_language_container_deployer(container_path=container_path, - language_alias=language_alias, - pyexasol_connection=pyexasol_connection, - bucketfs_config=bucketfs_config) - assert_udf_running(pyexasol_connection, language_alias) + deployer = create_container_deployer(container_path=container_path, + language_alias=language_alias, + pyexasol_connection=pyexasol_connection, + bucketfs_config=bucketfs_config) + deployer.deploy_container(True) + with connection_factory(exasol_config) as new_connection: + assert_udf_running(new_connection, language_alias, schema) + + +def test_language_container_deployer_alter_session( + request: FixtureRequest, + export_slc: ExportInfo, + pyexasol_connection: ExaConnection, + connection_factory: Callable[[config.Exasol], ExaConnection], + exasol_config: config.Exasol, + bucketfs_config: config.BucketFs, +): + test_name: str = request.node.name + schema = test_name + language_alias = f"PYTHON3_TE_{test_name.upper()}" + container_path = Path(export_slc.cache_file) + with revert_language_settings(pyexasol_connection): + create_schema(pyexasol_connection, schema) + deployer = create_container_deployer(container_path=container_path, + language_alias=language_alias, + pyexasol_connection=pyexasol_connection, + bucketfs_config=bucketfs_config) + deployer.upload_container() + with connection_factory(exasol_config) as new_connection: + deployer = create_container_deployer(container_path=container_path, + language_alias=language_alias, + pyexasol_connection=new_connection, + bucketfs_config=bucketfs_config) + deployer.activate_container(LanguageActivationLevel.Session, True) + assert_udf_running(new_connection, language_alias, schema) + + +def test_language_container_deployer_activation_fail( + request: FixtureRequest, + export_slc: ExportInfo, + pyexasol_connection: ExaConnection, + connection_factory: Callable[[config.Exasol], ExaConnection], + exasol_config: config.Exasol, + bucketfs_config: config.BucketFs, +): + test_name: str = request.node.name + schema = test_name + language_alias = f"PYTHON3_TE_{test_name.upper()}" + container_path = Path(export_slc.cache_file) + with revert_language_settings(pyexasol_connection): + create_schema(pyexasol_connection, schema) + deployer = create_container_deployer(container_path=container_path, + language_alias=language_alias, + pyexasol_connection=pyexasol_connection, + bucketfs_config=bucketfs_config) + deployer.deploy_container(True) + with connection_factory(exasol_config) as new_connection: + deployer = create_container_deployer(container_path=container_path, + language_alias=language_alias, + pyexasol_connection=new_connection, + bucketfs_config=bucketfs_config) + with pytest.raises(RuntimeError): + deployer.activate_container(LanguageActivationLevel.System, False) def create_schema(pyexasol_connection: ExaConnection, schema: str): @@ -37,22 +101,22 @@ def create_schema(pyexasol_connection: ExaConnection, schema: str): pyexasol_connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema};") -def assert_udf_running(pyexasol_connection: ExaConnection, language_alias: str): +def assert_udf_running(pyexasol_connection: ExaConnection, language_alias: str, schema: str): pyexasol_connection.execute(textwrap.dedent(f""" - CREATE OR REPLACE {language_alias} SCALAR SCRIPT "TEST_UDF"() + CREATE OR REPLACE {language_alias} SCALAR SCRIPT {schema}."TEST_UDF"() RETURNS BOOLEAN AS def run(ctx): return True / """)) - result = pyexasol_connection.execute('SELECT "TEST_UDF"()').fetchall() + result = pyexasol_connection.execute(f'SELECT {schema}."TEST_UDF"()').fetchall() assert result[0][0] == True -def call_language_container_deployer(container_path: Path, - language_alias: str, - pyexasol_connection: ExaConnection, - bucketfs_config: config.BucketFs): +def create_container_deployer(container_path: Path, + language_alias: str, + pyexasol_connection: ExaConnection, + bucketfs_config: config.BucketFs) -> LanguageContainerDeployer: bucket_fs_factory = BucketFSFactory() bucketfs_location = bucket_fs_factory.create_bucketfs_location( url=f"{bucketfs_config.url}/" @@ -61,6 +125,6 @@ def call_language_container_deployer(container_path: Path, user=f"{bucketfs_config.username}", pwd=f"{bucketfs_config.password}", base_path=None) - language_container_deployer = LanguageContainerDeployer( + return LanguageContainerDeployer( pyexasol_connection, language_alias, bucketfs_location, container_path) - language_container_deployer.deploy_container() + diff --git a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py index 3370c896..21c7de5c 100644 --- a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py +++ b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py @@ -174,13 +174,11 @@ def test_language_container_deployer_cli_with_check_cert( request: FixtureRequest, export_slc: ExportInfo, pyexasol_connection: ExaConnection, - connection_factory: Callable[[config.Exasol], ExaConnection], exasol_config: config.Exasol, bucketfs_config: config.BucketFs ): use_ssl_cert_validation = True - expected_exception_message = 'Could not connect to Exasol: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify ' \ - 'failed: self signed certificate in certificate chain (_ssl.c:1131)' + expected_exception_message = '[SSL: CERTIFICATE_VERIFY_FAILED]' test_name: str = request.node.name schema = test_name language_alias = f"PYTHON3_TE_{test_name.upper()}" @@ -198,5 +196,5 @@ def test_language_container_deployer_cli_with_check_cert( use_ssl_cert_validation=use_ssl_cert_validation) assert result.exit_code == 1 \ - and result.exception.args[0].message in expected_exception_message \ + and expected_exception_message in result.exception.args[0].message \ and type(result.exception) == ExaConnectionFailedError diff --git a/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py b/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py index 34ca0814..79ac0fa4 100644 --- a/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py +++ b/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py @@ -49,11 +49,10 @@ def test_scripts_deployer_cli_with_encryption_verify(language_alias: str, "--language-alias", language_alias, "--use-ssl-cert-validation" ] - expected_exception_message = 'Could not connect to Exasol: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify ' \ - 'failed: self signed certificate in certificate chain (_ssl.c:1131)' + expected_exception_message = '[SSL: CERTIFICATE_VERIFY_FAILED]' runner = CliRunner() result = runner.invoke(deploy.main, args_list) assert result.exit_code == 1 \ - and result.exception.args[0].message in expected_exception_message \ + and expected_exception_message in result.exception.args[0].message \ and type(result.exception) == ExaConnectionFailedError diff --git a/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py b/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py new file mode 100644 index 00000000..dc026ec1 --- /dev/null +++ b/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py @@ -0,0 +1,48 @@ +from pathlib import Path +from unittest.mock import create_autospec, MagicMock +import pytest +from pyexasol import ExaConnection +from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation +from exasol_transformers_extension.deployment.language_container_deployer import ( + LanguageContainerDeployer, LanguageActivationLevel) +from exasol_transformers_extension.deployment.language_container_deployer_cli import run_deployer + + +@pytest.fixture(scope='module') +def mock_pyexasol_conn() -> ExaConnection: + return create_autospec(ExaConnection) + + +@pytest.fixture(scope='module') +def mock_bfs_location() -> BucketFSLocation: + return create_autospec(BucketFSLocation) + + +@pytest.fixture +def container_deployer(mock_pyexasol_conn, mock_bfs_location) -> LanguageContainerDeployer: + return LanguageContainerDeployer(pyexasol_connection=mock_pyexasol_conn, + language_alias='alias', + bucketfs_location=mock_bfs_location, + container_file=Path('container_file')) + + +def test_language_container_deployer_cli_deploy(container_deployer): + container_deployer.deploy_container = MagicMock() + run_deployer(container_deployer, True, True, False) + container_deployer.deploy_container.assert_called_once_with(False) + + +def test_language_container_deployer_cli_upload(container_deployer): + container_deployer.upload_container = MagicMock() + container_deployer.activate_container = MagicMock() + run_deployer(container_deployer, True, False, False) + container_deployer.upload_container.assert_called_once() + container_deployer.activate_container.assert_not_called() + + +def test_language_container_deployer_cli_register(container_deployer): + container_deployer.upload_container = MagicMock() + container_deployer.activate_container = MagicMock() + run_deployer(container_deployer, False, True, True) + container_deployer.upload_container.assert_not_called() + container_deployer.activate_container.assert_called_once_with(LanguageActivationLevel.System, True)