diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 914e78f..e4e2ab2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -33,10 +33,11 @@ jobs: sudo luarocks install luaunit sudo luarocks install mockagne sudo luarocks install exaerror + sudo luarocks install luacheck + sudo luarocks install luacov - - name: Run Lua unit tests - run: | - ./scripts/run_lua_unittests.sh + - name: Run Lua unit tests and static code analyzer + run: poetry run poe lua-tests - name: Setup integration test environment run: ./scripts/setup_integration_test.sh diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml index 45c5cd8..008d75b 100644 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -38,10 +38,11 @@ jobs: sudo luarocks install luaunit sudo luarocks install mockagne sudo luarocks install exaerror + sudo luarocks install luacheck + sudo luarocks install luacov - - name: Run Lua unit tests - run: | - ./scripts/run_lua_unittests.sh + - name: Run Lua unit tests and static code analyzer + run: poetry run poe lua-tests - name: Setup integration test environment run: ./scripts/setup_integration_test.sh diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 9ad97b7..40f172a 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changelog +* [0.2.2](changes_0.2.2.md) * [0.2.1](changes_0.2.1.md) * [0.2.0](changes_0.2.0.md) * [0.1.0](changes_0.1.0.md) diff --git a/doc/changes/changes_0.2.2.md b/doc/changes/changes_0.2.2.md new file mode 100644 index 0000000..e57c10f --- /dev/null +++ b/doc/changes/changes_0.2.2.md @@ -0,0 +1,23 @@ +# SageMaker Extension 0.2.2, released 2021-12-17 + +Code name: Add static code analysis for Lua implementations to CI + +## Summary + +This release includes a linter and a coverage analyzer of the Lua implementation +to the CI pipeline of the SageMaker-Extension. Thus, the static code analysis of +Lua scripts are performed without executing them and, the coding standards and +conventions are adhered. + + +### Features + + - #48: Added static code analysis to CI for Lua + + + + + + + + diff --git a/exasol_sagemaker_extension/__init__.py b/exasol_sagemaker_extension/__init__.py index fb13a35..9dd16a3 100644 --- a/exasol_sagemaker_extension/__init__.py +++ b/exasol_sagemaker_extension/__init__.py @@ -1 +1 @@ -__version__ = '0.2.1' \ No newline at end of file +__version__ = '0.2.2' \ No newline at end of file diff --git a/exasol_sagemaker_extension/lua/src/autopilot_training_main.lua b/exasol_sagemaker_extension/lua/src/autopilot_training_main.lua index 06e7cdc..5c1ce2f 100644 --- a/exasol_sagemaker_extension/lua/src/autopilot_training_main.lua +++ b/exasol_sagemaker_extension/lua/src/autopilot_training_main.lua @@ -27,7 +27,7 @@ local required_args = { max_candidates = nil, model_info_schema_name = nil, model_info_table_name = nil, - objective = Nil, + objective = nil, aws_tags = nil } @@ -187,7 +187,7 @@ function get_table_columns(schema_name, table_name) _G.global_env.error(tostring(error_obj)) end - local col_names, col_types = {}, {} + local col_names, col_types = {}, {} for i=1, #res do col_names[#col_names+1] = res[i][1] col_types[#col_types+1] = res[i][2] diff --git a/exasol_sagemaker_extension/lua/src/db_metadata_reader.lua b/exasol_sagemaker_extension/lua/src/db_metadata_reader.lua index aaafa85..156dd98 100644 --- a/exasol_sagemaker_extension/lua/src/db_metadata_reader.lua +++ b/exasol_sagemaker_extension/lua/src/db_metadata_reader.lua @@ -39,7 +39,8 @@ function M.read_metadata(schema, job_name, columns) end end - local query_reading = [[SELECT ]] .. table.concat(columns_ids, ",") .. [[ FROM ::schema."SME_METADATA_AUTOPILOT_JOBS" WHERE JOB_NAME = :job_name]] + local query_reading = [[SELECT ]] .. table.concat(columns_ids, ",") + .. [[ FROM ::schema."SME_METADATA_AUTOPILOT_JOBS" WHERE JOB_NAME = :job_name]] local success, result = _G.global_env.pquery(query_reading, params) if not success then local error_obj = exaerror.create("F-SME-3", diff --git a/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deletion.lua b/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deletion.lua index ee393a7..da8f416 100644 --- a/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deletion.lua +++ b/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deletion.lua @@ -1,6 +1,6 @@ local luaunit = require("luaunit") local mockagne = require("mockagne") -require("src/autopilot_endpoint_deletion") +require("autopilot_endpoint_deletion") test_autopilot_endpoint_deletion = { diff --git a/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deployment.lua b/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deployment.lua index a0b641b..944847a 100644 --- a/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deployment.lua +++ b/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deployment.lua @@ -1,6 +1,6 @@ local luaunit = require("luaunit") local mockagne = require("mockagne") -require("src/autopilot_endpoint_deployment") +require("autopilot_endpoint_deployment") test_autopilot_endpoint_deployment = { diff --git a/exasol_sagemaker_extension/lua/test/test_autopilot_job_status_polling.lua b/exasol_sagemaker_extension/lua/test/test_autopilot_job_status_polling.lua index 82f0a18..8010730 100644 --- a/exasol_sagemaker_extension/lua/test/test_autopilot_job_status_polling.lua +++ b/exasol_sagemaker_extension/lua/test/test_autopilot_job_status_polling.lua @@ -1,6 +1,6 @@ local luaunit = require("luaunit") local mockagne = require("mockagne") -require("src/autopilot_job_status_polling") +require("autopilot_job_status_polling") test_autopilot_job_status_polling = { diff --git a/exasol_sagemaker_extension/lua/test/test_autopilot_training_main.lua b/exasol_sagemaker_extension/lua/test/test_autopilot_training_main.lua index 94cc873..b959370 100644 --- a/exasol_sagemaker_extension/lua/test/test_autopilot_training_main.lua +++ b/exasol_sagemaker_extension/lua/test/test_autopilot_training_main.lua @@ -1,6 +1,6 @@ local luaunit = require("luaunit") local mockagne = require("mockagne") -require("./src/autopilot_training_main") +require("autopilot_training_main") @@ -33,10 +33,10 @@ end function test_train_autopilot_main.test_parse_used_optional_arguments() local args_missing_json_str = [[{ - "job_name" : "jobname", - "aws_credentials_connection_name" : "S3_CONNECTION", - "aws_region" : "eu-central-1", - "iam_sagemaker_role" : "arn:aws:iam::1234:role", + "job_name" : "jobname", + "aws_credentials_connection_name" : "S3_CONNECTION", + "aws_region" : "eu-central-1", + "iam_sagemaker_role" : "arn:aws:iam::1234:role", "s3_bucket_uri" : "s3://sme-bucket", "s3_output_path" : "path", "input_schema_name" : "TEST_SCHEMA", diff --git a/exasol_sagemaker_extension/lua/test/test_aws_s3_handler.lua b/exasol_sagemaker_extension/lua/test/test_aws_s3_handler.lua index 09a7838..4967de4 100644 --- a/exasol_sagemaker_extension/lua/test/test_aws_s3_handler.lua +++ b/exasol_sagemaker_extension/lua/test/test_aws_s3_handler.lua @@ -1,6 +1,6 @@ local luaunit = require("luaunit") local mockagne = require("mockagne") -local aws_s3_handler = require("./src/aws_s3_handler") +local aws_s3_handler = require("aws_s3_handler") test_aws_s3_handler = { @@ -67,7 +67,7 @@ end function test_aws_s3_handler.test_prepare_export_query() local parallelism_factor = 1 - for i = 1, 5 do + for _ = 1, 5 do local n_nodes = math.random(1, 20) local n_exporter = n_nodes * parallelism_factor diff --git a/exasol_sagemaker_extension/lua/test/test_aws_sagemaker_handler.lua b/exasol_sagemaker_extension/lua/test/test_aws_sagemaker_handler.lua index d8da180..2af8bd3 100644 --- a/exasol_sagemaker_extension/lua/test/test_aws_sagemaker_handler.lua +++ b/exasol_sagemaker_extension/lua/test/test_aws_sagemaker_handler.lua @@ -1,6 +1,6 @@ local luaunit = require("luaunit") local mockagne = require("mockagne") -local aws_sagemaker_handler = require("./src/aws_sagemaker_handler") +local aws_sagemaker_handler = require("aws_sagemaker_handler") test_aws_sagemaker_handler = { diff --git a/exasol_sagemaker_extension/lua/test/test_db_metadata_reader.lua b/exasol_sagemaker_extension/lua/test/test_db_metadata_reader.lua index 6f27fe1..60da800 100644 --- a/exasol_sagemaker_extension/lua/test/test_db_metadata_reader.lua +++ b/exasol_sagemaker_extension/lua/test/test_db_metadata_reader.lua @@ -1,6 +1,6 @@ local luaunit = require("luaunit") local mockagne = require("mockagne") -local db_metadata_reader = require("./src/db_metadata_reader") +local db_metadata_reader = require("db_metadata_reader") test_db_metadata_reader = { diff --git a/exasol_sagemaker_extension/lua/test/test_db_metadata_writer.lua b/exasol_sagemaker_extension/lua/test/test_db_metadata_writer.lua index 6de425c..ef347c9 100644 --- a/exasol_sagemaker_extension/lua/test/test_db_metadata_writer.lua +++ b/exasol_sagemaker_extension/lua/test/test_db_metadata_writer.lua @@ -1,6 +1,6 @@ local luaunit = require("luaunit") local mockagne = require("mockagne") -local db_metadata_writer = require("./src/db_metadata_writer") +local db_metadata_writer = require("db_metadata_writer") test_db_metadata_writer = { diff --git a/exasol_sagemaker_extension/lua/test/test_endpoint_connection_handler.lua b/exasol_sagemaker_extension/lua/test/test_endpoint_connection_handler.lua index a2f4fb3..aa7fd93 100644 --- a/exasol_sagemaker_extension/lua/test/test_endpoint_connection_handler.lua +++ b/exasol_sagemaker_extension/lua/test/test_endpoint_connection_handler.lua @@ -1,7 +1,7 @@ local json = require('cjson') local luaunit = require("luaunit") local mockagne = require("mockagne") -local endpoint_connection_handler = require("src/endpoint_connection_handler") +local endpoint_connection_handler = require("endpoint_connection_handler") local aws_s3_connection = 'aws_s3_connection' diff --git a/exasol_sagemaker_extension/lua/test/test_install_autopilot_prediction_udf.lua b/exasol_sagemaker_extension/lua/test/test_install_autopilot_prediction_udf.lua index d9105d9..187ddb7 100644 --- a/exasol_sagemaker_extension/lua/test/test_install_autopilot_prediction_udf.lua +++ b/exasol_sagemaker_extension/lua/test/test_install_autopilot_prediction_udf.lua @@ -1,7 +1,7 @@ local luaunit = require("luaunit") local mockagne = require("mockagne") local install_autopilot_prediction_udf = require( - "./src/install_autopilot_prediction_udf") + "install_autopilot_prediction_udf") test_install_autopilot_prediction_udf = {} diff --git a/exasol_sagemaker_extension/lua/test/test_validate_input.lua b/exasol_sagemaker_extension/lua/test/test_validate_input.lua index 58905eb..c0dedde 100644 --- a/exasol_sagemaker_extension/lua/test/test_validate_input.lua +++ b/exasol_sagemaker_extension/lua/test/test_validate_input.lua @@ -1,5 +1,5 @@ local luaunit = require("luaunit") -local validate_input = require("./src/validate_input") +local validate_input = require("validate_input") test_validate_input= { diff --git a/pyproject.toml b/pyproject.toml index f8d9ce9..790ba9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "exasol-sagemaker-extension" -version = "0.2.1" +version = "0.2.2" description = "Exasol SageMaker Integration" license = "MIT" @@ -20,7 +20,7 @@ keywords = ['exasol', 'sagemaker'] include = ["lua", "resources", "deployment"] [tool.poetry.dependencies] -python = ">=3.6.1,<4.0" +python = ">=3.6.2,<4.0" pandas = ">=1.1.3,<2.0.0" boto = "^2.49.0" sagemaker = "^2.59.1" @@ -30,12 +30,18 @@ importlib-resources = "^5.2.0" [tool.poetry.dev-dependencies] pytest = "^5.2" -pytest-cov = "^2.12.1" +pytest-cov = "^2.10.1" Sphinx = "^4.1.2" coverage = "^5.5" exasol-udf-mock-python = { git = "https://github.com/exasol/udf-mock-python.git", branch = "master" } exasol-bucketfs-utils-python = { git = "https://github.com/exasol/bucketfs-utils-python.git", branch = "main" } +poethepoet = "^0.10.0" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" + + +[tool.poe.tasks] +lua-tests = {"shell" = "bash ./scripts/lua_tests.sh"} + diff --git a/scripts/lua_tests.sh b/scripts/lua_tests.sh new file mode 100755 index 0000000..38a5744 --- /dev/null +++ b/scripts/lua_tests.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +# This script finds and runs Lua unit tests, collects coverage and runs static code analysis. + +readonly script_dir=$(dirname "$(readlink -f "$0")") +if [[ -v $1 ]] +then + readonly base_dir="$1" +else + readonly base_dir=$(readlink -f "$script_dir/..") +fi + +readonly exit_ok=0 +readonly exit_software=2 +readonly src_module_path="$base_dir/exasol_sagemaker_extension/lua/src" +readonly test_module_path="$base_dir/exasol_sagemaker_extension/lua/test" +readonly target_dir="$base_dir/target" +readonly reports_dir="$target_dir/luaunit-reports" +readonly luacov_dir="$target_dir/luacov-reports" + +function create_target_directories { + mkdir -p "$reports_dir" + mkdir -p "$luacov_dir" +} + +## +# Run the unit tests and collect code coverage. +# +# Return error status in case there were failures. +# +function run_tests { + cd "$test_module_path" || exit + readonly tests="$(find . -name '*.lua')" + test_suites=0 + failures=0 + successes=0 + for testcase in $tests + do + ((test_suites++)) + + if LUA_PATH="$src_module_path/?.lua;$(luarocks path --lr-path)" \ + lua -lluacov "$testcase" -o junit -n "$reports_dir/$testcase" + then + ((successes++)) + else + ((failures++)) + fi + echo + done + echo -n "Ran $test_suites test suites. $successes successes, " + if [[ "$failures" -eq 0 ]] + then + echo -e "\e[1m\e[32m$failures failures\e[0m." + return "$exit_ok" + else + echo -e "\e[1m\e[31m$failures failures\e[0m." + return "$exit_software" + fi +} + +## +# Collect the coverage results into a single file. +# +# Return exit status of coverage collector. +# +function collect_coverage_results { + echo + echo "Collecting code coverage results" + luacov + return "$?" +} + +## +# Move the coverage results into the target directory. +# +# Return exit status of `mv` command. +# +function move_coverage_results { + echo "Moving coverage results to $luacov_dir" + mv "$test_module_path"/luacov.*.out "$luacov_dir" + return "$?" +} + +## +# Print the summary section of the code coverage report to the console +# +function print_coverage_summary { + echo + grep --after 500 'File\s*Hits' "$luacov_dir/luacov.report.out" +} + +## +# Analyze the Lua code with "luacheck". +# +# Return exit status of code coverage. +# +function run_static_code_analysis { + echo + echo "Running static code analysis" + echo + luacheck "$src_module_path" "$test_module_path" --codes \ + --ignore 111 --ignore 112 --ignore 113 --ignore 142 + return "$?" +} + +create_target_directories +run_tests \ +&& collect_coverage_results \ +&& move_coverage_results \ +&& print_coverage_summary \ +&& run_static_code_analysis \ +|| exit "$exit_software" + +exit "$exit_ok" \ No newline at end of file diff --git a/scripts/run_lua_unittests.sh b/scripts/run_lua_unittests.sh deleted file mode 100755 index 754da1e..0000000 --- a/scripts/run_lua_unittests.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# This scripts run all Lua scripts implemented in lua/test directory - - -readonly script_dir=$(dirname "$(readlink -f "$0")") -readonly base_dir=$(readlink -f "$script_dir/..") -readonly lua_dir="$base_dir/exasol_sagemaker_extension/lua" - - -if ! cd "$lua_dir" -then - echo "Lua scripts path is not correct: " $lua_dir - exit 2 -fi - -n_test=0 -n_success=0 -n_fail=0 -readonly tests="$(find './test/' -name '*.lua')" -for unittest in $tests -do - ((n_test+=1)) - if lua "$unittest" -v - then - ((n_success+=1)) - else - ((n_fail+=1)) - fi - echo "" -done - -echo -n "Ran $n_test tests, $n_success successes, $n_fail failures" -exit 0 \ No newline at end of file diff --git a/setup.py b/setup.py index 8c3c0dd..2bbdb2e 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup_kwargs = { 'name': 'exasol-sagemaker-extension', - 'version': '0.2.1', + 'version': '0.2.2', 'description': 'Exasol SageMaker Integration', 'long_description': '# SageMaker Extension\n\nThis project provides a Python library that trains data stored in Exasol using AWS SageMaker.\n\n\n## Table of Contents\n\n### Information for Users\n\n* [User Guide](doc/user_guide/user_guide.md)\n* [Tutorial](https://github.com/exasol/data-science-examples/blob/main/tutorials/machine-learning/sagemaker-extension/tutorial.md)\n* [Changelog](doc/changes/changelog.md)\n\n### Information for Contributors\n\n\n* [System Requirement Specification](doc/system_requirements.md)\n* [Design](doc/design.md)\n* [Dependencies](doc/dependencies.md)', 'author': 'Umit Buyuksahin', @@ -31,7 +31,7 @@ 'packages': packages, 'package_data': package_data, 'install_requires': install_requires, - 'python_requires': '>=3.6.1,<4.0', + 'python_requires': '>=3.6.2,<4.0', } diff --git a/tests/unit_tests/test_sagemaker_extension.py b/tests/unit_tests/test_sagemaker_extension.py index 23da44a..66125b4 100644 --- a/tests/unit_tests/test_sagemaker_extension.py +++ b/tests/unit_tests/test_sagemaker_extension.py @@ -2,4 +2,4 @@ def test_version(): - assert __version__ == '0.2.1' \ No newline at end of file + assert __version__ == '0.2.2' \ No newline at end of file