From ccb50e112ec0970704933432d8c0a49ad6c6066f Mon Sep 17 00:00:00 2001 From: MarleneKress79789 Date: Tue, 17 Oct 2023 15:32:53 +0200 Subject: [PATCH 1/2] fixed language container download --- doc/changes/changes_0.6.0.md | 26 ++++++++++++ doc/user_guide/user_guide.md | 4 +- .../deployment/deployment_utils.py | 42 ++++++------------- .../deployment/language_container_deployer.py | 4 +- .../utils/dataframe_operations.py | 2 +- .../test_language_container_deployer_cli.py | 2 +- 6 files changed, 45 insertions(+), 35 deletions(-) create mode 100644 doc/changes/changes_0.6.0.md diff --git a/doc/changes/changes_0.6.0.md b/doc/changes/changes_0.6.0.md new file mode 100644 index 00000000..f7d2136c --- /dev/null +++ b/doc/changes/changes_0.6.0.md @@ -0,0 +1,26 @@ +# Transformers Extension 0.6.0, released T.B.D + +Code name: T.B.D + + +## Summary + +T.B.D + +### Features + + - n/a + +### Bug Fixes + + - # 134: Fixed slc download still assuming existence of slc-parts + +### Refactorings + + - n/a + +### Documentation + + - n/a + + \ No newline at end of file diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index be2aaecb..899e5c7e 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -83,7 +83,7 @@ pip install transformers_extension.whl --extra-index-url https://download.pytorc ### The Pre-built Language Container This extension requires the installation of the language container for this -extension to run. It can be installed in two ways:Quick and Customized +extension to run. It can be installed in two ways: Quick and Customized installations #### Quick Installation @@ -147,7 +147,7 @@ There are two ways to install the language container: (1) using a python script --bucket \ --path-in-bucket \ --language-alias \ - --container-file + --container-file ``` 2. *Manual Installation* diff --git a/exasol_transformers_extension/deployment/deployment_utils.py b/exasol_transformers_extension/deployment/deployment_utils.py index 910c352e..b1b04e5b 100644 --- a/exasol_transformers_extension/deployment/deployment_utils.py +++ b/exasol_transformers_extension/deployment/deployment_utils.py @@ -1,12 +1,9 @@ -import os -import glob + import logging import requests import tempfile -import subprocess import ssl from pathlib import Path -from getpass import getpass from contextlib import contextmanager from jinja2 import Environment, PackageLoader, select_autoescape from exasol_transformers_extension.deployment import constants @@ -14,11 +11,10 @@ logger = logging.getLogger(__name__) + DB_PASSWORD_ENVIRONMENT_VARIABLE = f"TE_DB_PASSWORD" BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE = f"TE_BUCKETFS_PASSWORD" -SLC_PARTS_PREFIX_NAME = 'language_container_part_0' -SLC_FINAL_NAME = "language_container.tar.gz" -N_SLC_PARTS = 2 +SLC_NAME = "exasol_transformers_extension_container_release.tar.gz" GH_RELEASE_URL = "https://github.com/exasol/transformers-extension/releases/download" @@ -31,25 +27,14 @@ def load_and_render_statement(template_name, **kwargs) -> str: return statement -def _download_slc_parts(tmp_dir, version): - for i in range(N_SLC_PARTS): - slc_part_name = f"{SLC_PARTS_PREFIX_NAME}" + str(i) - url = "/".join((GH_RELEASE_URL, version, slc_part_name)) - response = requests.get(url, stream=True) - response.raise_for_status() - with open(Path(tmp_dir, slc_part_name), 'wb') as f: - f.write(response.content) - - -def _concatenate_slc_parts(tmp_dir): - slc_final_path = Path(tmp_dir, SLC_FINAL_NAME) - slc_parts_path = glob.glob(str(Path(tmp_dir, SLC_PARTS_PREFIX_NAME)) + "*") - cmd = f"cat {' '.join(slc_parts_path)} > {slc_final_path}" - subprocess.run(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True) - return slc_final_path +def _download_slc(tmp_dir: Path, version: str) -> Path: + url = "/".join((GH_RELEASE_URL, version, SLC_NAME)) + response = requests.get(url, stream=True) + response.raise_for_status() + slc_path = Path(tmp_dir, SLC_NAME) + with open(slc_path, 'wb') as f: + f.write(response.content) + return slc_path def get_websocket_ssl_options(use_ssl_cert_validation: bool, ssl_cert_path: str): @@ -65,9 +50,8 @@ def get_websocket_ssl_options(use_ssl_cert_validation: bool, ssl_cert_path: str) @contextmanager -def get_container_file_from_github_release(version): +def get_container_file_from_github_release(version: str): with tempfile.TemporaryDirectory() as tmp_dir: - _download_slc_parts(tmp_dir, version) - container_file_path = _concatenate_slc_parts(tmp_dir) + container_file_path = _download_slc(tmp_dir, version) yield container_file_path diff --git a/exasol_transformers_extension/deployment/language_container_deployer.py b/exasol_transformers_extension/deployment/language_container_deployer.py index 1f13faa3..5fecd024 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer.py +++ b/exasol_transformers_extension/deployment/language_container_deployer.py @@ -48,8 +48,8 @@ def _generate_alter_command(self, alter_type: str, f"ALTER {alter_type} SET SCRIPT_LANGUAGES='{new_settings}';" return alter_command - def _update_previous_language_settings( - self, alter_type: str, path_in_udf: PurePosixPath) -> str: + def _update_previous_language_settings(self, alter_type: str, + path_in_udf: PurePosixPath) -> str: prev_lang_settings = self._get_previous_language_settings(alter_type) prev_lang_aliases = prev_lang_settings.split(" ") self._check_if_requested_language_alias_already_exists( diff --git a/exasol_transformers_extension/utils/dataframe_operations.py b/exasol_transformers_extension/utils/dataframe_operations.py index 97a44dea..a0669889 100644 --- a/exasol_transformers_extension/utils/dataframe_operations.py +++ b/exasol_transformers_extension/utils/dataframe_operations.py @@ -10,7 +10,7 @@ def get_unique_values( Filter given dataframe and return the unique values. Sorts the unique values if the sort parameter is set to True - :param df: Dataframe from which the uniques values are extracted + :param df: Dataframe from which the unique values are extracted :param columns: List of columns to be filtered and sorted accordingly :param sort: Sort the unique values by given columns if True """ diff --git a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py index 474705d7..ae016766 100644 --- a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py +++ b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py @@ -125,7 +125,7 @@ def test_language_container_deployer_cli_by_downloading_container( schema = test_name language_alias = f"PYTHON3_TE_{test_name.upper()}" container_path = None - version = "0.2.0" + version = "0.5.0" create_schema(pyexasol_connection, schema) dsn = f"{exasol_config.host}:{exasol_config.port}" with revert_language_settings(pyexasol_connection): From a6d703950ee49a97180389ee79adb2565c38cb0c Mon Sep 17 00:00:00 2001 From: MarleneKress79789 Date: Tue, 17 Oct 2023 15:34:47 +0200 Subject: [PATCH 2/2] [CodeBuild] --- .../with_db/deployment/test_language_container_deployer_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py index ae016766..3370c896 100644 --- a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py +++ b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py @@ -200,4 +200,3 @@ def test_language_container_deployer_cli_with_check_cert( assert result.exit_code == 1 \ and result.exception.args[0].message in expected_exception_message \ and type(result.exception) == ExaConnectionFailedError -