diff --git a/.github/actions/prepare_poetry_env/action.yml b/.github/actions/prepare_poetry_env/action.yml new file mode 100644 index 00000000..2bf84c74 --- /dev/null +++ b/.github/actions/prepare_poetry_env/action.yml @@ -0,0 +1,19 @@ +name: 'Prepare Poetry environment' +description: 'This composite actions checks out out the project, installs Poetry, and install the project in the Poetry environment' +inputs: + python-version: + description: 'The Python version to use' + required: true + default: '3.8' +runs: + using: "composite" + steps: + - uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.python-version }} + - uses: abatilo/actions-poetry@v2 + with: + poetry-version: 1.4.0 + - name: Poetry install + run: poetry install + shell: bash diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b7a04eac..7a4ff89c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,16 +13,13 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - uses: abatilo/actions-poetry@v2.1.4 - with: - poetry-version: 1.2.0 - - name: Install Poetry - run: poetry install + + - name: Setup Python & Poetry Environment + uses: ./.github/actions/prepare_poetry_env + - name: Build Poetry run: poetry build + - name: Install project whl run: pip install dist/*.whl diff --git a/.github/workflows/check_setup_py.yaml b/.github/workflows/check_setup_py.yaml deleted file mode 100644 index fabb5343..00000000 --- a/.github/workflows/check_setup_py.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Check if setup.py is up to date - -on: [push] - -jobs: - check_setup_py: - strategy: - fail-fast: false - matrix: - python-version: [3.8] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - uses: abatilo/actions-poetry@v2.1.4 - with: - poetry-version: 1.2.0 - - name: Poetry install - run: poetry install - - name: Run packaging update - run: bash githooks/update_setup_py.sh - - name: Show changes on working copy - run: git status --porcelain=v1 -uno - - name: Show diff on working copy - run: git diff --cached; cat setup.py - - name: Check if setup.py changed - run: | - [ -z "$(git status --porcelain=v1 -uno 2>/dev/null)" ] - - name: Check if setup.py works - run: pip install . diff --git a/.github/workflows/language_container.yaml b/.github/workflows/language_container.yaml index 09a474d8..8d448bd6 100644 --- a/.github/workflows/language_container.yaml +++ b/.github/workflows/language_container.yaml @@ -10,19 +10,19 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - uses: abatilo/actions-poetry@v2.1.4 - with: - poetry-version: 1.2.0 + + - name: Setup Python & Poetry Environment + uses: ./.github/actions/prepare_poetry_env + - name: Build language container run: ./build_language_container.sh + - name: Archive language container uses: actions/upload-artifact@v2 with: name: exasol_sagemaker_extension_container path: .build_output/cache/exports/*.tar.gz* + - name: Archive language container build logs uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml index 496ff10d..5b157f46 100644 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -17,16 +17,13 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - uses: abatilo/actions-poetry@v2.1.4 - with: - poetry-version: 1.2.0 - - name: Install Poetry - run: poetry install + + - name: Setup Python & Poetry Environment + uses: ./.github/actions/prepare_poetry_env + - name: Build Poetry run: poetry build + - name: Install project whl run: pip install dist/*.whl diff --git a/ci-isolation/pom.xml b/ci-isolation/pom.xml index 3d45cd2b..6e0c47d2 100644 --- a/ci-isolation/pom.xml +++ b/ci-isolation/pom.xml @@ -38,7 +38,7 @@ com.exasol ci-isolation-aws - 2.0.0 + 2.0.1 \ No newline at end of file diff --git a/ci-isolation/src/main/java/com/exasol/adapter/document/files/ciisolation/CiIsolationApp.java b/ci-isolation/src/main/java/com/exasol/adapter/document/files/ciisolation/CiIsolationApp.java index 54861059..31443ec4 100644 --- a/ci-isolation/src/main/java/com/exasol/adapter/document/files/ciisolation/CiIsolationApp.java +++ b/ci-isolation/src/main/java/com/exasol/adapter/document/files/ciisolation/CiIsolationApp.java @@ -2,6 +2,7 @@ import com.exasol.ciisolation.aws.PolicyReader; import com.exasol.ciisolation.aws.ciuser.CiUserStack; +import com.exasol.ciisolation.aws.ciuser.CiUserStack.CiUserStackProps; import software.amazon.awscdk.App; @@ -12,10 +13,11 @@ public class CiIsolationApp { public static void main(final String[] args) { final App app = new App(); final PolicyReader policyReader = new PolicyReader(); - new CiUserStack(app, CiUserStack.CiUserStackProps.builder().projectName("exasol-sagemaker-extension") + CiUserStackProps props = CiUserStack.CiUserStackProps.builder().projectName("exasol-sagemaker-extension") .addRequiredPermissions( policyReader.readPolicyFromResources("s3-access.json"), - policyReader.readPolicyFromResources("sagemaker-access.json")).build()); + policyReader.readPolicyFromResources("sagemaker-access.json")).build(); + new CiUserStack(app, props); app.synth(); } } diff --git a/githooks/pre-commit b/githooks/pre-commit index 15364a56..f9672b54 100755 --- a/githooks/pre-commit +++ b/githooks/pre-commit @@ -7,7 +7,6 @@ REPO_DIR=$(git rev-parse --show-toplevel) GITHOOKS_PATH="$REPO_DIR/githooks" pushd "$REPO_DIR" bash "$GITHOOKS_PATH/prohibit_commit_to_main.sh" -bash "$GITHOOKS_PATH/update_setup_py.sh" popd diff --git a/githooks/update_setup_py.sh b/githooks/update_setup_py.sh deleted file mode 100755 index 58d8182b..00000000 --- a/githooks/update_setup_py.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - -# define colors for use in output -green='\033[0;32m' -no_color='\033[0m' -grey='\033[0;90m' - -# Jump to the current project's root directory (the one containing -# .git/) -ROOT_DIR=$(git rev-parse --show-toplevel || echo) -NO_GIT=FALSE -if [ -z "$ROOT_DIR" ] -then - echo "Did not found git repository, using '$PWD' as ROOT_DIR" - NO_GIT=TRUE - ROOT_DIR=$PWD -fi - - -pushd "$ROOT_DIR" > /dev/null - -echo -e "Generate setup.py ${grey}(pre-commit hook)${no_color}" -if [ -d "dist" ] -then - rm -r "dist" -fi -poetry build > /dev/null -pushd dist > /dev/null -tar_file=$(ls *.tar.gz) -extracted_dir=${tar_file%.tar.gz} -tar -xf $tar_file -cp "$extracted_dir/setup.py" ../setup.py -rm -r "$extracted_dir" -popd > /dev/null - -if [ "$NO_GIT" == "FALSE" ] -then - echo -e "Add generated files ${grey}(pre-commit hook)${no_color}" - git add setup.py -fi - -popd > /dev/null diff --git a/pyproject.toml b/pyproject.toml index 88bce9cd..e574752a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,8 @@ include = ["lua", "resources", "deployment"] [tool.poetry.dependencies] python = ">=3.8,<4.0" pandas = ">=1.4.2,<2.0.0" -boto = "^2.49.0" +boto3 = "1.26.163" +botocore = "1.29.163" protobuf = ">=3.1,<=3.20.0" sagemaker = "^2.59.1" pyexasol = "^0.24.0" diff --git a/scripts/setup_integration_test.sh b/scripts/setup_integration_test.sh index fba27da0..08191df5 100755 --- a/scripts/setup_integration_test.sh +++ b/scripts/setup_integration_test.sh @@ -20,7 +20,7 @@ function install_localstack { function checkout_exasol_test_container { cnt_func=$((cnt_func+1)) echo -e "${YEL} Step-$cnt_func: ${GRA} Checkout Exasol test container ${NC}" - git clone https://github.com/exasol/integration-test-docker-environment.git + git clone --branch 1.6.0 https://github.com/exasol/integration-test-docker-environment.git } function spawn_exasol_environment { diff --git a/setup.py b/setup.py deleted file mode 100644 index 96589668..00000000 --- a/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -from setuptools import setup - -packages = \ -['exasol_sagemaker_extension', - 'exasol_sagemaker_extension.autopilot_utils', - 'exasol_sagemaker_extension.deployment'] - -package_data = \ -{'': ['*'], - 'exasol_sagemaker_extension': ['lua/src/*', - 'lua/test/*', - 'resources/lua/outputs/*', - 'resources/lua/templates/*', - 'resources/sql/*', - 'resources/udf/*']} - -install_requires = \ -['boto>=2.49.0,<3.0.0', - 'click>=8.0.3,<9.0.0', - 'importlib-resources>=5.2.0,<6.0.0', - 'localstack-client>=1.25,<2.0', - 'pandas>=1.4.2,<2.0.0', - 'protobuf>=3.1,<=3.20.0', - 'pyexasol>=0.24.0,<0.25.0', - 'sagemaker>=2.59.1,<3.0.0', - 'typeguard>=2.11.1,<3.0.0'] - -setup_kwargs = { - 'name': 'exasol-sagemaker-extension', - 'version': '0.4.0', - 'description': 'Exasol SageMaker Integration', - 'long_description': '# SageMaker Extension\n\nThis project provides a Python library that trains data stored in Exasol using AWS SageMaker.\n\n\n## Table of Contents\n\n### Information for Users\n\n* [User Guide](doc/user_guide/user_guide.md)\n* [Tutorial](https://github.com/exasol/data-science-examples/blob/main/tutorials/machine-learning/sagemaker-extension/tutorial.md)\n* [Changelog](doc/changes/changelog.md)\n\n### Information for Contributors\n\n\n* [System Requirement Specification](doc/system_requirements.md)\n* [Design](doc/design.md)\n* [Dependencies](doc/dependencies.md)', - 'author': 'Umit Buyuksahin', - 'author_email': 'umit.buyuksahin@exasol.com', - 'maintainer': 'None', - 'maintainer_email': 'None', - 'url': 'https://github.com/exasol/sagemaker-extension', - 'packages': packages, - 'package_data': package_data, - 'install_requires': install_requires, - 'python_requires': '>=3.8,<4.0', -} - - -setup(**setup_kwargs)